import matplotlib.pyplot as plt
import numpy as np
import torch
from scipy import special

class Resampler(torch.nn.Module):
    """
    Efficiently resample audio signals
    This module is much faster than resampling with librosa because it exploits pytorch's efficient conv1d operations
    This module is also faster than the existing pytorch resample function in
    https://github.com/pytorch/audio/blob/b6a61c3f7d0267c77f8626167cc1eda0335f2753/torchaudio/compliance/kaldi.py#L892

    Based on
    https://github.com/danpovey/filtering/blob/master/lilfilter/resampler2.py
    with improvements to include additional filter types and input parameters that align with the librosa api
    """

    def __init__(self, input_sr, output_sr, dtype, num_zeros=64, cutoff_ratio=0.95, filter='kaiser', beta=14.0):
        super().__init__()  # init the base class
        """This creates an object that can apply a symmetric FIR filter based
        on torch.nn.functional.conv1d.

        Args:
          input_sr:  The input sampling rate, AS AN INTEGER..
              does not have to be the real sampling rate but should
              have the correct ratio with output_sr.
          output_sr:  The output sampling rate, AS AN INTEGER.
              It is the ratio with the input sampling rate that is
              important here.
          dtype:  The torch dtype to use for computations (would be preferable to
               set things up so passing the dtype isn't necessary)
          num_zeros: The number of zeros per side in the (sinc*hanning-window)
              filter function.  More is more accurate, but 64 is already
              quite a lot. The kernel size is 2*num_zeros + 1.
          cutoff_ratio: The filter rolloff point as a fraction of the
             Nyquist frequency.
          filter: one of ['kaiser', 'kaiser_best', 'kaiser_fast', 'hann']
          beta: parameter for 'kaiser' filter

        You can think of this algorithm as dividing up the signals
        (input,output) into blocks where there are `input_sr` input
        samples and `output_sr` output samples.  Then we treat it
        using convolutional code, imagining there are `input_sr`
        input channels and `output_sr` output channels per time step.
        """
        assert isinstance(input_sr, int) and isinstance(output_sr, int)
        if input_sr == output_sr:
            self.resample_type = 'trivial'
            return

        def gcd(a, b):
            """Return the greatest common divisor of a and b."""
            assert isinstance(a, int) and isinstance(b, int)
            if b == 0:
                return a
            else:
                return gcd(b, a % b)

        d = gcd(input_sr, output_sr)
        input_sr, output_sr = input_sr // d, output_sr // d

        assert dtype in [torch.float32, torch.float64]
        assert num_zeros > 3  # a reasonable bare minimum
        np_dtype = np.float32 if dtype == torch.float32 else np.float64

        assert filter in ['hann', 'kaiser', 'kaiser_best', 'kaiser_fast']

        if filter == 'kaiser_best':
            num_zeros = 64
            beta = 14.769656459379492
            cutoff_ratio = 0.9475937167399596
            filter = 'kaiser'
        elif filter == 'kaiser_fast':
            num_zeros = 16
            beta = 8.555504641634386
            cutoff_ratio = 0.85
            filter = 'kaiser'

        # Define one 'block' of samples `input_sr` input samples
        # and `output_sr` output samples.  We can divide up
        # the samples into these blocks and have the blocks be
        # in correspondence.

        # The sinc function will have, on average, `zeros_per_block`
        # zeros per block.
        zeros_per_block = min(input_sr, output_sr) * cutoff_ratio

        # The convolutional kernel size will be n = (blocks_per_side*2 + 1),
        # i.e. we add that many blocks on each side of the central block.  The
        # window radius (defined as distance from center to edge)
        # is `blocks_per_side` blocks.  This ensures that each sample in the
        # central block can "see" all the samples in its window.
        #
        # Assuming the following division is not exact, adding 1
        # will have the same effect as rounding up.
        # blocks_per_side = 1 + int(num_zeros / zeros_per_block)
        blocks_per_side = int(np.ceil(num_zeros / zeros_per_block))

        kernel_width = 2 * blocks_per_side + 1

        # We want the weights as used by torch's conv1d code; format is
        #  (out_channels, in_channels, kernel_width)
        # https://pytorch.org/docs/stable/nn.functional.html
        weights = torch.tensor((output_sr, input_sr, kernel_width), dtype=dtype)

        # Computations involving time will be in units of 1 block.  Actually this
        # is the same as the `canonical` time axis since each block has input_sr
        # input samples, so it would be one of whatever time unit we are using
        window_radius_in_blocks = blocks_per_side

        # The `times` below will end up being the args to the sinc function.
        # For the shapes of the things below, look at the args to `view`.  The terms
        # below will get expanded to shape (output_sr, input_sr, kernel_width) through
        # broadcasting
        # We want it so that, assuming input_sr == output_sr, along the diagonal of
        # the central block we have t == 0.
        # The signs of the output_sr and input_sr terms need to be opposite.  The
        # sign that the kernel_width term needs to be will depend on whether it's
        # convolution or correlation, and the logic is tricky.. I will just find
        # which sign works.

        times = (
            np.arange(output_sr, dtype=np_dtype).reshape((output_sr, 1, 1)) / output_sr
            - np.arange(input_sr, dtype=np_dtype).reshape((1, input_sr, 1)) / input_sr
            - (np.arange(kernel_width, dtype=np_dtype).reshape((1, 1, kernel_width)) - blocks_per_side)
        )

        def hann_window(a):
            """hann_window returns the Hann window on [-1,1], which is zero if
            a < -1 or a > 1, and otherwise 0.5 + 0.5 cos(a*pi). This is applied
            elementwise to a, which should be a NumPy array.

            The heaviside function returns (a > 0 ? 1 : 0).
            """
            return np.heaviside(1 - np.abs(a), 0.0) * (0.5 + 0.5 * np.cos(a * np.pi))

        def kaiser_window(a, beta):
            w = special.i0(beta * np.sqrt(np.clip(1 - ((a - 0.0) / 1.0) ** 2.0, 0.0, 1.0))) / special.i0(beta)
            return np.heaviside(1 - np.abs(a), 0.0) * w

        # The weights below are a sinc function times a Hann-window function.
        #
        # Multiplication by zeros_per_block normalizes the sinc function
        # (to compensate for scaling on the x-axis), so that the integral is 1.
        #
        # Division by input_sr normalizes the input function. Think of the input
        # as a stream of dirac deltas passing through a low pass filter:
        # in order to have the same magnitude as the original input function,
        # we need to divide by the number of those deltas per unit time.
        if filter == 'hann':
            weights = (
                np.sinc(times * zeros_per_block)
                * hann_window(times / window_radius_in_blocks)
                * zeros_per_block
                / input_sr
            )
        else:
            weights = (
                np.sinc(times * zeros_per_block)
                * kaiser_window(times / window_radius_in_blocks, beta)
                * zeros_per_block
                / input_sr
            )

        self.input_sr = input_sr
        self.output_sr = output_sr

        # weights has dim (output_sr, input_sr, kernel_width).
        # If output_sr == 1, we can fold the input_sr into the
        # kernel_width (i.e. have just 1 input channel); this will make the
        # convolution faster and avoid unnecessary reshaping.

        assert weights.shape == (output_sr, input_sr, kernel_width)
        if output_sr == 1:
            self.resample_type = 'integer_downsample'
            self.padding = input_sr * blocks_per_side
            weights = torch.tensor(weights, dtype=dtype, requires_grad=False)
            self.weights = weights.transpose(1, 2).contiguous().view(1, 1, input_sr * kernel_width)

        elif input_sr == 1:
            # In this case we'll be doing conv_transpose, so we want the same weights that
            # we would have if we were *downsampling* by this factor-- i.e. as if input_sr,
            # output_sr had been swapped.
            self.resample_type = 'integer_upsample'
            self.padding = output_sr * blocks_per_side
            weights = torch.tensor(weights, dtype=dtype, requires_grad=False)
            self.weights = weights.flip(2).transpose(0, 2).contiguous().view(1, 1, output_sr * kernel_width)
        else:
            self.resample_type = 'general'
            self.reshaped = False
            self.padding = blocks_per_side
            self.weights = torch.tensor(weights, dtype=dtype, requires_grad=False)

        self.weights = torch.nn.Parameter(self.weights, requires_grad=False)

    @torch.no_grad()
    def forward(self, data):
        """Resample the data.

        Args:
         input: a torch.Tensor with the same dtype as was passed to the
           constructor.
         There must be 2 axes, interpreted as (minibatch_size, sequence_length)...
         the minibatch_size may in practice be the number of channels.

        Return:  Returns a torch.Tensor with the same dtype as the input, and
         dimension (minibatch_size, (sequence_length//input_sr)*output_sr),
         where input_sr and output_sr are the corresponding constructor args,
         modified to remove any common factors.
        """
        if self.resample_type == 'trivial':
            return data
        elif self.resample_type == 'integer_downsample':
            (minibatch_size, seq_len) = data.shape
            # will be shape (minibatch_size, in_channels, seq_len) with in_channels == 1
            data.unsqueeze_(1)
            data = torch.nn.functional.conv1d(data, self.weights, stride=self.input_sr, padding=self.padding)
            # shape will be (minibatch_size, out_channels = 1, seq_len);
            # return as (minibatch_size, seq_len)
            return data.squeeze_(1)

        elif self.resample_type == 'integer_upsample':
            data.unsqueeze_(1)
            data = torch.nn.functional.conv_transpose1d(data, self.weights, stride=self.output_sr, padding=self.padding)

            return data.squeeze_(1)
        else:
            assert self.resample_type == 'general'
            (minibatch_size, seq_len) = data.shape
            num_blocks = seq_len // self.input_sr
            if num_blocks == 0:
                # TODO: pad with zeros.
                raise RuntimeError('Signal is too short to resample')
            # data = data[:, 0:(num_blocks*self.input_sr)]  # Truncate input
            data = data[:, 0 : (num_blocks * self.input_sr)].view(minibatch_size, num_blocks, self.input_sr)

            # Torch's conv1d expects input data with shape (minibatch, in_channels, time_steps), so transpose
            data.transpose_(1, 2)

            data = torch.nn.functional.conv1d(data, self.weights, padding=self.padding)

            assert data.shape == (minibatch_size, self.output_sr, num_blocks)
            return data.transpose(1, 2).contiguous().view(minibatch_size, num_blocks * self.output_sr)

if __name__ == '__main__':
    fs = 100
    fs_ = 128
    R = Resampler(fs, fs_, dtype=torch.float32)
    f = 10
    t = torch.arange(-1 / 2, 1 / 2, 1 / fs)
    x = torch.sin(2 * np.pi * f * t) / (2 * np.pi * f * t)
    x[x.shape[0] // 2] = 1
    t_ = torch.arange(-1 / 2, 1 / 2, 1 / fs_)
    x = x.view(1, -1)
    y = R(x)
    # plt.subplot(2,1,1)
    plt.plot(t, x.view(-1), '.-')
    # plt.subplot(2,1,2)
    plt.plot(t_, y.view(-1))
    plt.grid()
    plt.show()


#!/usr/bin/env python3

def main():
    count = 0
    with open("dracula.txt", "r") as file:
        with open("vampytimes.txt", "w") as vampirefile:
            for line in file:
                if "vampire" in line.lower():
                    print(line)
                    count += 1
                    vampirefile.write(line)
    print(count)

if __name__ == "__main__":
    main()


class Solution(object):
    # def findMin(self, nums):
    #     """
    #     :type nums: List[int]
    #     :rtype: int
    #     """
    #     return self.get_min(nums, 0, len(nums) - 1)
    #
    # def get_min(self, nums, start, end):
    #     mid = (start + end) / 2
    #     if start == end:
    #         # one element
    #         return nums[start]
    #     if mid == start or mid == end:
    #         # two element
    #         return min(nums[start], nums[end])
    #     if nums[mid] < nums[end]:
    #         # right side sorted
    #         if nums[mid] > nums[start]:
    #             # not rotated
    #             return nums[start]
    #         return self.get_min(nums, start, mid)
    #     elif nums[mid] > nums[end]:
    #         # left side sorted
    #         return self.get_min(nums, mid, end)

    def findMin(self, nums):
        # A[l] > A[r]
        l, r = 0, len(nums) - 1
        while l < r and nums[l] >= nums[r]:
            mid = (l + r) / 2
            if nums[mid] > nums[r]:
                l = mid + 1
            else:
                r = mid
        return nums[l]


import os

import cv2
import math
import numpy as np

def preprocessing(img,num):
    print("pre processing...")
    image= img
    # gray convertion
    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    grad_x = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=3)
    grad_y = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=3)

    abs_grad_x = cv2.convertScaleAbs(grad_x)
    abs_grad_y = cv2.convertScaleAbs(grad_y)

    # threshold
    thresh_x = cv2.threshold(abs_grad_x, 0, 255, cv2.THRESH_OTSU)[1]
    med_x = cv2.medianBlur(thresh_x, 3)
    thresh_y = cv2.threshold(abs_grad_y, 0, 255, cv2.THRESH_OTSU)[1]
    med_y = cv2.medianBlur(thresh_y, 3)
    #cv2.imwrite('x.jpg',abs_grad_x)
    #cv2.imwrite('y.jpg', abs_grad_y)

    # bluring
    kernel_size = 3
    blur_thresh_x = cv2.GaussianBlur(med_x, (kernel_size, kernel_size), 0)
    blur_thresh_y = cv2.GaussianBlur(med_y, (kernel_size, kernel_size), 0)
    #cv2.imwrite('xg.jpg', blur_thresh_x)
    #cv2.imwrite('yg.jpg', blur_thresh_y)
    # Run Hough on edge detected image
    width = image.shape[1]
    height = image.shape[0]
    rho = 1  # distance resolution in pixels of the Hough grid
    theta_vertical = np.pi  # angular resolution in radians of the Hough grid
    theta_horizonatal = np.pi / 2
    threshold_horizontal = math.floor(width * 0.2)  # minimum number of votes (intersections in Hough grid cell)
    threshold_vertical = math.floor(height * 0.2)
    min_line_length_horizontal = math.floor(width * 0.2)
    min_line_length_vertical = math.floor(height * 0.2)  # minimum number of pixels making up a line
    max_line_gap = 1  # maximum gap in pixels between connectable line segments
    line_image = np.copy(gray) * 0  # creating a blank to draw lines on

    # Vertical lines
    vertical_lines = cv2.HoughLinesP(blur_thresh_x, rho, theta_vertical, threshold_vertical, np.array([]),
                                     min_line_length_vertical, max_line_gap)

    if vertical_lines is not None:
        for line in vertical_lines:
            for x1, y1, x2, y2 in line:
                # here it's possible to add a selection of only vertical lines
                if np.abs(y1 - y2) > 0.1 * np.abs(x1 - x2):
                    cv2.line(line_image, (x1, y1), (x2, y2), 255, 5)

    # Horizontal lines
    horizontal_lines = cv2.HoughLinesP(blur_thresh_y, rho, theta_horizonatal, threshold_horizontal, np.array([]),
                                       min_line_length_horizontal, max_line_gap)

    if horizontal_lines is not None:
        for line in horizontal_lines:
            for x1, y1, x2, y2 in line:
                # here it's possible to add a selection of only horizontal lines
                if np.abs(x1 - x2) > 0.1 * np.abs(y1 - y2):
                    cv2.line(line_image, (x1, y1), (x2, y2), 255, 5)

    # threshold
    thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1]
    #thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY_INV, 61,18)

    # remove lines
    clean_thresh = cv2.subtract(thresh, line_image)
    clean_invert = cv2.bitwise_not(clean_thresh)
    #cv2.imwrite('l.jpg',line_image)
    cv2.namedWindow('detectedLines', cv2.WINDOW_NORMAL)
    cv2.imshow('detectedLines', line_image)
    cv2.namedWindow('preProcessed', cv2.WINDOW_NORMAL)
    cv2.imshow('preProcessed', clean_invert)
    cv2.waitKey(0)
    name = "C:/Users/Shashi/Documents/FYP/FYP_Final/Outputs/" + num
    cv2.imwrite(os.path.join(name,'pp.jpg'), clean_invert)
    return  clean_invert


from unittest.mock import patch

from test.test_stubbed_youtube import TestStubbedYouTube

class IdToTitleTest(TestStubbedYouTube):
    @patch('internal.operations.report_mappings')
    def test_operation(self, reports):
        self._stub_as_existing_in_playlist('the_video_id',
                                           'the_title',
                                           'the_playlist_id')
        self._invoke('id-to-title the_playlist_id')
        reports.assert_called_with({
                'the_video_id': 'the_title'})


from datetime import datetime

from abstract_currency import AbstractCurrency
from abstract_date import AbstractDate, START_DATE, END_DATE, OBSERVATIONS

class MonthEnd(AbstractDate):

    def __init__(self, date_to_lookup: datetime):
        super().__init__(date_to_lookup)

    def get_exchange_rate(self, currency: AbstractCurrency):
        year = str(self._date.year)
        bank_of_canada_response = currency.get_bank_of_canada_response(START_DATE + year + '-01-01' + END_DATE +
                                                                       year + '-12-31')
        first_available_year = bank_of_canada_response[OBSERVATIONS][0]['d'].split('-')[0]
        last_available_year = bank_of_canada_response[OBSERVATIONS][-1]['d'].split('-')[0]

        if first_available_year > year or last_available_year < year:
            print('No month end data available for ' + year)
        else:
            for i in range(1, 13):
                rates_for_months = list(filter(lambda x: x['d'].startswith(f'{year}-{i:02}'),
                                               bank_of_canada_response[OBSERVATIONS]))

                if len(rates_for_months):
                    AbstractDate._print_rate(rates_for_months[-1])
                else:
                    break


import argparse
import os
import re
import requests  # Install with easy_install or pip install
import shutil
import subprocess
import sys
import tempfile
from contextlib import contextmanager
from updatecommon import get_release
from updatecommon import upload_release

BOTTLE_TARGET = 'yosemite_or_later'

@contextmanager
def os_closing(o):
    yield o
    os.close(o)

def parse_args(args):
    parser = argparse.ArgumentParser(description='Update homebrew-fb')
    parser.add_argument(
        'version_tag',
        help='The name of the tag to create a release for')
    parser.add_argument(
        'github_token',
        type=file,
        help='The authentication token to use to talk to GitHub')
    parser.add_argument(
        '--tap-repo-location',
        default='/usr/local/Library/Taps/facebook/homebrew-fb',
        help='The location of the homebrew-fb tap')
    return parser.parse_args(args)

def build_bottle(version_tag, tap_repo_location):
    print('Building bottle...')
    subprocess.check_call(
        [
            'brew',
            'unlink',
            'buck',
        ],
        cwd=tap_repo_location)
    subprocess.check_call(
        [
            'brew',
            'install',
            '--build-bottle',
            'buck',
        ],
        cwd=tap_repo_location)
    subprocess.check_call(
        [
            'brew',
            'bottle',
            'buck',
        ],
        cwd=tap_repo_location)
    dest_name = 'buck-{version_name}.{bottle_target}.bottle.tar.gz'.format(
        version_name=version_tag[1:],
        bottle_target=BOTTLE_TARGET)
    subprocess.check_call(
        [
            'mv',
            'buck-{version_name}.el_capitan.bottle.tar.gz'.format(
                version_name=version_tag[1:]),
            dest_name,
        ],
        cwd=tap_repo_location)
    return os.path.join(tap_repo_location, dest_name)

def fetch_tarball(url):
    print('Fetching tarball from `{url}`...'.format(url=url))
    r = requests.get(url, stream=True)
    handle, path = tempfile.mkstemp(suffix='.tar.gz')
    with os_closing(handle) as handle:
        for chunk in r.iter_content(chunk_size=4096):
            if chunk:
                os.write(handle, chunk)
    return path

def validate():
    print('Validating bottle...')
    subprocess.check_call([
        'brew',
        'uninstall',
        '--force',
        'buck',
    ])
    subprocess.check_call([
        'brew',
        'install',
        'buck',
    ])

def update_bottle(version_tag, github_token, tap_repo_location):
    release_data = get_release(version_tag)
    tarball_sha256 = sha256(fetch_tarball(release_data['tarball_url']))

    # First, update the bottle to have the new version and tarball sha.
    temp_handle, temp_path = tempfile.mkstemp(text=True)
    with os_closing(temp_handle):
        with open(os.path.join(tap_repo_location, 'buck.rb'), 'r') as orig:
            for line in orig:
                line = re.sub(
                    r'@@buck_version = .*$',
                    '@@buck_version = "{version_name}"'.format(
                        version_name=version_tag[1:]),
                    line)
                line = re.sub(
                    r'sha256 "[a-z0-9]{64}"$',
                    'sha256 "{sha}"'.format(sha=tarball_sha256),
                    line)
                line = re.sub(
                    r'  url "https://.+"$',
                    '  url "{url}"'.format(url=release_data['tarball_url']),
                    line)
                os.write(temp_handle, line)
        shutil.copyfile(temp_path, os.path.join(tap_repo_location, 'buck.rb'))

    # Now, build the bottle's binary, and update the file with the new sha.
    bottle_file = build_bottle(version_tag, tap_repo_location)
    bottle_sha256 = sha256(bottle_file)
    upload_release(
            bottle_file,
            release_data['upload_url'],
            github_token,
            {
                'Content-Type': 'application/x-tar',
            })
    os.remove(bottle_file)

    temp_handle, temp_path = tempfile.mkstemp(text=True)
    with os_closing(temp_handle):
        with open(os.path.join(tap_repo_location, 'buck.rb'), 'r') as orig:
            for line in orig:
                line = re.sub(
                    r'sha256 "[a-z0-9]{64}" => :.+$',
                    'sha256 "{sha}" => :{bottle_target}'.format(
                        sha=bottle_sha256,
                        bottle_target=BOTTLE_TARGET),
                    line)
                os.write(temp_handle, line)
        shutil.copyfile(temp_path, os.path.join(tap_repo_location, 'buck.rb'))

    validate()

    subprocess.check_call(
        [
            'git',
            'commit',
            '-m',
            'Update `buck.rb` to {version_tag}'.format(
                version_tag=version_tag),
            'buck.rb',
        ],
        cwd=tap_repo_location)
    print('Your commit is ready for testing!  Check it out:')
    print(tap_repo_location)

def sha256(file_name):
    return subprocess.check_output([
        'shasum',
        '-a',
        '256',
        file_name,
    ]).split()[0]

if __name__ == '__main__':
    args = parse_args(sys.argv[1:])
    github_token_lines = args.github_token.readlines()
    assert len(github_token_lines) == 1, (
        'Invalid number of lines in github_token file')
    update_bottle(
        args.version_tag,
        github_token_lines[0].strip(),
        args.tap_repo_location)


# OPENCORE - ADD
from methods.regular.regular_api import *
from methods.task.task_template.task_template_after_launch_strategies.task_template_after_launch_strategy import \
    TaskTemplateAfterLaunchStrategy
from shared.connection.connection_strategy import ConnectionStrategy
from methods.task.task_template.task_template_after_launch_strategies.standard_task_template_after_launch_strategy import \
    StandardTaskTemplateAfterLaunchStrategy
from shared.database.external.external import ExternalMap
from shared.regular.regular_log import result_has_error

class LabelboxTaskTemplateAfterLaunchStrategy(TaskTemplateAfterLaunchStrategy):
    LABELBOX_DIFFGRAM_TOOLS_MAPPING = {
        'box': 'rectangle',
        'polygon': 'polygon'
    }
    # Label attribute --> classification mapping
    LABELBOX_DIFFGRAM_CLASSIFICATIONS_MAPPING = {
        'multiple_select': 'checklist',
        'radio': 'radio',
        'text': 'text',
        'select': 'dropdown'
    }

    def __create_tools_from_task_templates(self):
        result = []
        for label_element in self.task_template.label_dict['label_file_list_serialized']:

            element = {
                'name': f"{label_element['label']['name']}",
                'color': label_element['colour']['hex'].upper(),
                'tool': self.LABELBOX_DIFFGRAM_TOOLS_MAPPING[self.task_template.instance_type],
                'required': False,

            }
            """
                Attribute format for labelbox classifications(24/8/2020).:
                'classifications': [{'required': True,
                                     'instructions': 'radio classiff',
                                     'name': 'radio_classiff',
                                     'type': 'radio',
                                     'options': [{'label': 'test1', 'value': 'test1'},
                                                {'label': 'test2', 'value': 'test2'},
                                                {'label': 'test3', 'value': 'test3'}]},
                                    {'required': True,
                                     'instructions': 'checklist classifd',
                                     'name': 'checklist_classifd',
                                     'type': 'checklist',
                                     'options': [{'label': 'option1', 'value': 'option1'},
                                                    {'label': 'option2', 'value': 'option2'},
                                                    {'label': 'option3', 'value': 'option3'}]},
                                    {'required': True,
                                     'instructions': 'test classficactiin: question?',
                                     'name': 'test_classficactiin:_question?',
                                     'type': 'text',
                                     'options': []},
                                    {'required': True,
                                     'instructions': 'dropdown',
                                     'name': 'dropdown',
                                     'type': 'dropdown',
                                     'options': [{'label': 'dropdown op1', 'value': 'dropdown_op1'},
                                                 {'label': 'dropdown opt2', 'value': 'dropdown_opt2'},
                                                 {'label': 'dropdown opt3', 'value': 'dropdown_opt3'}]}]}]
            """
            # Add attribute groups as nested classifications for labelbox.
            if label_element.get('attribute_group_list') and len(label_element.get('attribute_group_list')) > 0:
                attribute_groups = label_element.get('attribute_group_list')
                element['classifications'] = []
                for attribute in attribute_groups:
                    group = {
                        'required': False,
                        'name': attribute.get('name'),
                        'instructions': attribute.get('prompt'),
                        'type': self.LABELBOX_DIFFGRAM_CLASSIFICATIONS_MAPPING[attribute.get('kind')],
                        'options': [
                            {'label': opt.get('name'), 'value': opt.get('name')}
                            for opt in attribute['attribute_template_list']
                        ]
                    }
                    element['classifications'].append(group)
            result.append(element)
        return result

    def save_label_instance_ontology_mapping(self, ontology, connection):
        """
            Saves the relationships betweend ID's of Diffgram label instances
            and ID's of the featureNode ID's in the onthology.
        :param ontology:
        :return:
        """
        mapping = {}
        tools = ontology['project']['ontology']['normalized']['tools']
        for tool in tools:
            diffgram_label_file = self.task_template.get_label_file_by_name(tool['name'])
            diffgram_label_file_id = self.task_template.get_label_file_by_name(tool['name'])['id']
            # Feature schema ID was removed from API. Using name instead
            # feature_schema_id = tool['featureSchemaId']
            feature_schema_id = tool['name']
            mapping[feature_schema_id] = {'label_id': diffgram_label_file_id, 'attributes': {}}
            ExternalMap.new(session=self.session,
                            file_id=diffgram_label_file_id,
                            external_id=feature_schema_id,
                            type=connection.integration_name,
                            diffgram_class_string='label_file',
                            connection=connection,
                            add_to_session=True)
            if tool.get('classifications', None):
                classifications = tool.get('classifications', None)
                for classification in classifications:
                    attribute_group = self.task_template.get_attribute_group_by_name(diffgram_label_file,
                                                                                classification['name'])
                    diffgram_attribute_group_id = attribute_group['id']
                    # feature_schema_id = classification['featureSchemaId']
                    # Changing to name since feature schema was removed.
                    feature_schema_id = classification['name']
                    ExternalMap.new(session=self.session,
                                    external_id=feature_schema_id,
                                    file_id=diffgram_label_file_id,
                                    attribute_template_group_id=diffgram_attribute_group_id,
                                    type=connection.integration_name,
                                    diffgram_class_string='label_file',
                                    connection=connection,
                                    add_to_session=True)
        return mapping

    def create_labelbox_project(self, connector):

        result = connector.put_data({
            'action_type': 'create_project',
            'name': self.task_template.name,
            'event_data': {},
        })
        if result_has_error(result):
            raise Exception(f"Failed to setup labelbox project. {str(result)}")
        if 'result' in result:
            labelbox_project = result['result']
        else:
            logger.error(result)
        return labelbox_project

    def create_labelbox_project_mapping(self, labelbox_project, connection):
        external_map = ExternalMap.new(
            session=self.session,
            job=self.task_template,
            external_id=labelbox_project.uid,
            connection=connection,
            diffgram_class_string='task_template',
            type=connection.integration_name,
            url=f"https://app.labelbox.com/projects/{labelbox_project.uid}/overview",
            add_to_session=True,
            flush_session=True
        )
        # Commented to bottom to avoid circular dependencies on job.
        self.task_template.default_external_map = external_map

        logger.debug(f"Created Labelbox Project {labelbox_project.uid}")
        return external_map

    def get_labelbox_frontend(self, connector):
        result = connector.fetch_data({
            'action_type': 'get_default_frontend',
            'event_data': {},
        })
        if result_has_error(result):
            raise Exception(f"Failed to setup labelbox frontend. {str(result)}")
        frontend = result['result']
        logger.debug('Created Labelbox Frontend.')
        return frontend

    def create_ontology(self, connector, frontend, labelbox_project):
        options = {"tools": self.__create_tools_from_task_templates(), "classifications": []}
        options = json.dumps(options)
        result = connector.put_data({
            'action_type': 'setup_ontology',
            'frontend': frontend,
            'ontology': options,
            'project': labelbox_project,
            'event_data': {},
        })
        if result_has_error(result):
            raise Exception(f"Failed to setup labelbox onthology. {str(result)}")
        logger.debug('Created Labelbox Project Ontology.')
        # Now save ontology relations
        query_ontology = """
            query($projectId: ID!) {
                project (where: {id: $projectId }) {
                    ontology {
                        normalized
                    }
                }
            }
        """
        data = {'projectId': labelbox_project.uid}
        result_ontology = connector.put_data({'action_type': 'execute',
                                              'query': query_ontology,
                                              'data': data,
                                              'event_data': {}})
        ontology = result_ontology['result']
        return ontology

    def execute_after_launch_strategy(self):
        """
            This strategy will attach files from sync directories and creates tasks in
            Diffgram for each of them.
        :return:
        """
        labelbox_project = None
        try:
            # TODO: ADD LABELBOX LOGIC HERE
            # We don't check perms here because we assume this was checked on the task template creation.
            # Otherwise, we would need request context here, which we don't have.
            connection = self.task_template.interface_connection
            logger.debug(f"Connection for labelbox: {connection}")
            connector, success = ConnectionStrategy(connection=connection, session=self.session).get_connector()
            connector.connect()

            # First we create a project
            labelbox_project = self.create_labelbox_project(connector)
            external_map = self.create_labelbox_project_mapping(labelbox_project, connection)

            # Now create labelbox frontend
            frontend = self.get_labelbox_frontend(connector)

            # Next, we specify an ontology based on the label_templates of the task template.
            # Note: for now all classifications wil be nested.
            ontology = self.create_ontology(connector, frontend, labelbox_project)
            self.save_label_instance_ontology_mapping(ontology, connection)

            # Then we create a dataset for the project, where we'll add all the files
            labelbox_sync_manager = LabelBoxSyncManager(
                session=self.session,
                labelbox_project=labelbox_project,
                task_template=self.task_template,
                log=self.log,
                labelbox_connector=connector
            )

            # We first set the webhook for this task template.
            labelbox_sync_manager.set_webhook_for_task_template()

            # Then we start adding all files to labelbox.
            labelbox_sync_manager.send_all_files_in_task_template()
            logger.info(
                'LabelboxTaskTemplateAfterLaunchStrategy for Task Template ID: {} completed successfully.'.format(
                    self.task_template.id))
            logger.debug('Proceding to standard task template launch...')

            # Now create tasks as usual.
            standard_strategy = StandardTaskTemplateAfterLaunchStrategy(
                session=self.session,
                task_template=self.task_template,
                log=self.log
            )
            standard_strategy.execute_after_launch_strategy()

        except Exception as e:
            logger.exception(e)
            # Rollback Delete Labelbox Project
            if labelbox_project is not None:
                if self.task_template.default_external_map:
                    self.task_template.default_external_map = None
                    self.session.delete(self.task_template.default_external_map)
                labelbox_project.delete()
                logger.debug(f"ROLLBACK. Deleted Labelbox Project {labelbox_project.uid}")
                # Allow time for rate limiter in case of a rate limit exception.
                time.sleep(5)
            raise e


import socket
HOST = ''              # Endereco IP do Servidor
PORT = 5000            # Porta que o Servidor esta
PORT_FB = 4000

orig = (HOST, PORT)
fb = (HOST, PORT_FB)

udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
udp.bind(orig)

while True:
    msg, cliente = udp.recvfrom(1024)
    print(cliente, msg.decode())

    udp.sendto((msg.decode() + "_fb").encode(), fb)

udp.close()


import numpy as np
import math
import random
from collections import deque
from shapely.geometry import Point, LineString
from trajectory.rrt import RRT

def is_saw(x, y, n):
    '''
    Checks if walk of length n is self-avoiding
    :param (x,y)(list,list): walk of length n
    :param n(int): length of the walk
    :return: True if the walk is self-avoiding
    '''
    # creating a set removes duplicates, so it suffices to check the size of the set
    return n+1 == len(set(zip(x, y)))

def in_map(x, y):
    '''
    Checks if walk is in map
    :param (x,y)(list,list): the walk
    :return judge: True if the walk is in map
    '''
    judge = False
    x, y = np.array(x), np.array(y)
    if (abs(x) < 9.5).all() and (abs(y) < 9.5).all():
        judge = True
    return judge

def myopic_saw(n, x_init, y_init):
    '''
    Tries to generate a SAW of length n using myopic algorithm
    :param n(int): length of walk
    :param (x_init, y_init)(float, float): initial coordinate of saw
    :return (x,y)(list,list): the walk (length <= n)
    '''
    theta = [i * math.pi / 2 for i in range(0, 4)]
    r = 5    # original one: 0.75
    x, y = [x_init], [y_init]
    positions = set([(x_init,y_init)])    # positions is a set(no same element) that stores all sites visited
    for i in range(n):
        t = random.choice(theta)
        x_new = x[-1] + r * round(math.cos(t), 2)
        y_new = y[-1] + r * round(math.sin(t), 2)
        if (x_new, y_new) not in positions:
            x.append(x_new)
            y.append(y_new)
            positions.add((x_new, y_new))
        else:
            continue
    return x, y

def dimer(n, x_init, y_init):
    '''
    Generates a SAW by dimerization
    :param n(int): length of walk
    :param (x_init, y_init)(float, float): initial coordinate of saw
    :return (x_concat,y_concat)(list,list): walk of length n
    '''
    if n <= 3:
        x, y = myopic_saw(n, x_init, y_init)
        return x, y
    else:
        not_saw = True
        while not_saw:
            x_1, y_1 = dimer(n//2, x_init, y_init)
            x_2, y_2 = dimer(n - n//2, x_init, y_init)
            x_2 = [(x_1[-1] - x_1[0] + x) for x in x_2]
            y_2 = [(y_1[-1] - y_1[0] + y) for y in y_2]
            x_concat, y_concat = x_1 + x_2[1:], y_1 + y_2[1:]
            if is_saw(x_concat, y_concat, n) and in_map(x_concat, y_concat):
                not_saw = False
        return x_concat, y_concat

def interpolation(x, y, n_trajectory):
    '''
    Generates continous trajectory by whittaker-shannon interpolation formula
    :param (x,y)(list,list): discrete self-avoiding walk
    :param n_trajectory(int): length of continous trajectory
    :return (x_contin,y_contin)(list,list): continous trajectory
    '''
    interval = 1
    N = len(x)
    t = np.linspace(0, (N - 1) * interval, n_trajectory)
    x_contin, y_contin = 0, 0
    for n in range(N):
        x_contin += x[n] * np.sinc(t / interval - n)
        y_contin += y[n] * np.sinc(t / interval - n)
    return x_contin, y_contin

def bezier_smooth(traj_points, n_trajectory):
    raw_trajectory = np.array(traj_points)
    raw_traj_x = raw_trajectory[:, 0]
    raw_traj_y = raw_trajectory[:, 1]

    N = raw_trajectory.shape[0] - 1
    t = np.linspace(0, 1, n_trajectory)
    polynomial = []
    for i in range(N+1):
        polynomial.append(math.comb(N, i) * (t ** (N - i)) * (1 - t) ** i)

    polynomial = np.array(polynomial)
    traj_x = np.dot(raw_traj_x, polynomial)
    traj_y = np.dot(raw_traj_y, polynomial)
    return list(zip(traj_x, traj_y))

def gen_saw_track(x_init, y_init, n_trajectory=1100, n_discre=69):
    '''
    Generates continous self-avoiding trajectory
    :param (x_init, y_init)(float, float): initial coordinate of saw
    :param n_discre(int): length of discrete self-avoiding walk
    :param n_trajectory(int): length of continous trajectory
    :return trajectory(list): continous trajectory
    '''
    x, y = dimer(n_discre, x_init, y_init)
    trajectory = bezier_smooth(list(zip(x, y)), n_trajectory)
    return trajectory

def gen_rose_track(x_init, y_init, n_trajectory=1200):
    x, y = [], []
    for i in range(n_trajectory):
        k = i * np.pi / 400
        r = 5 * np.sin(4 * k)
        x.append(0.5 * x_init + r * np.cos(k))
        y.append(0.5 * y_init + r * np.sin(k))
    trajectory = list(zip(x, y))
    return trajectory

def gen_spiral_track(x_init, y_init, n_trajectory=1200):
    x, y = [], []
    for i in range(n_trajectory):
        k = i * math.pi / 90
        r = 0.12 * k
        x.append(0.5 * x_init + r * math.cos(k))
        y.append(0.5 * y_init + r * math.sin(k))
    trajectory = list(zip(x, y))
    return trajectory

def gen_circle_track(x_init, y_init, r=10, n_trajectory=1010):
    x, y = [], []
    for i in range(n_trajectory):
        k = i * math.pi / 90
        x.append(x_init + r * math.cos(k))
        y.append(y_init + r * math.sin(k))
    trajectory = list(zip(x, y))
    return trajectory

def gen_square_track(x_init, y_init, a=19, r=6):
    '''
    number of trajectory points is 1012
    '''
    x, y = [], []
    i = 0
    center = [x_init+(a/2-r), y_init+(a/2-r)]
    while i < 150:
        k = i * math.pi / 300
        x.append(center[0] + r * math.cos(k))
        y.append(center[1] + r * math.sin(k))
        i += 1
    for j in range(103):
        x.append(x[-1] - (a-2*r)/100)
        y.append(y[-1])

    center = [x_init-(a/2-r), y_init+(a/2-r)]
    while i < 300:
        k = i * math.pi / 300
        x.append(center[0] + r * math.cos(k))
        y.append(center[1] + r * math.sin(k))
        i += 1
    for j in range(103):
        x.append(x[-1])
        y.append(y[-1] - (a-2*r)/100)

    center = [x_init-(a/2-r), y_init-(a/2-r)]
    while i < 450:
        k = i * math.pi / 300
        x.append(center[0] + r * math.cos(k))
        y.append(center[1] + r * math.sin(k))
        i += 1
    for j in range(103):
        x.append(x[-1] + (a-2*r)/100)
        y.append(y[-1])

    center = [x_init+(a/2-r), y_init-(a/2-r)]
    while i < 600:
        k = i * math.pi / 300
        x.append(center[0] + r * math.cos(k))
        y.append(center[1] + r * math.sin(k))
        i += 1
    for j in range(103):
        x.append(x[-1])
        y.append(y[-1] + (a-2*r)/100)

    trajectory = list(zip(x, y))
    return trajectory

def gen_random_rrt_track(x_init, y_init, n_trajectory, env_size, poly_list):
    p_start = [x_init, y_init]
    step_len = 0.5
    random_path_len = 11   # eval: 10-100-400    env3: 5-50-200
    self_avoid_poly = deque([Point(-10, -10), Point(-10, -10), Point(-10, -10)], maxlen=3)
    poly_list.extend(self_avoid_poly)
    overall_path = []
    while len(overall_path) < 400:
        rrt = RRT(p_start, (0, 0), step_len, random_path_len=random_path_len)
        random_path = rrt.random_planning(env_size, poly_list)
        if not random_path:
            return None
        new_path = bezier_smooth(random_path, 100)
        overall_path.extend(new_path)

        self_avoid_poly.append(LineString(new_path[:-10]))
        poly_list[-3:] = self_avoid_poly
        p_start = overall_path[-1]

    trajectory = bezier_smooth(overall_path, n_trajectory)
    trajectory.reverse()
    return trajectory

def gen_simple_rrt_track(multi_goal_list, n_trajectory, env_size, poly_list):
    step_len = 0.5
    rrt = RRT(multi_goal_list[0], multi_goal_list[1], step_len)
    path1 = rrt.path_planning(env_size, poly_list)
    rrt = RRT(multi_goal_list[1], multi_goal_list[2], step_len)
    path2 = rrt.path_planning(env_size, poly_list)
    if not path1 or not path2:
        return None
    overall_path = bezier_smooth(path1, 200)
    overall_path.extend(bezier_smooth(path2, 200))
    trajectory = bezier_smooth(overall_path, n_trajectory)
    trajectory.reverse()
    return trajectory


import os
import difflib
import subprocess

from web import app

@app.context_processor
def inject_grader():
    def run_grade(stale):
        dummy = open("sandbox/foo.py", "w+")
        dummy.write(stale)
        dummy.close()

        subprocess.call("python -m black sandbox/foo.py", shell=True)

        dummy = open("sandbox/foo.py", "r")
        f = dummy.read()
        diff = difflib.SequenceMatcher(None, stale, f)
        dummy.close()
        os.remove("sandbox/foo.py")

        r = round(diff.ratio(), 2) * 100

        if r >= 90:
            color = "#008000"
        elif r >= 80:
            color = "#ffff00"
        else:
            color = "#ff0000"

        return [str(r), color, str(f)]

    return dict(run_grade=run_grade)


from random import randint

population=30
mutation_chance=10

def getScores(s_weight,s_value,s_maxweight,s_gen):
        """
        calculates the scores for the different treasures
        input: weight, value, maxweight (for the bag) and the generation
        output: Scores for each part of the generation containing weight and value for the bag
        """
        s_results=[]

        for i in range(len(s_gen)):
                s_total_mass=0
                s_total_value=0

                for j in range(len(s_gen[i])):
                        s_total_mass+=s_gen[i][j]*s_weight[j]
                        s_total_value+=s_gen[i][j]*s_value[j]

                if s_total_mass>s_maxweight:
                        s_total_value=0

                s_results+=[[s_total_mass,s_total_value]]

        return s_results

def get_super_performer(gsp_gen,gsp_scores):
        """
        Get the highest performing individual in the population
        input: the generation and their scores
        output: the index and score of the highest scoring individual
        """
        gsp_index=0
        gsp_max=0
        for i in range(len(gsp_scores)):
                if gsp_scores[i][1]>gsp_max:
                        gsp_max=gsp_scores[i][1]
                        gsp_index=i
        return gsp_index,gsp_max

def mutant_clone(mc_gen,scores):
        """
        Change the population depending on the highest scoring individual
        input: the generation, their relative scores
        output: A new generation in a list,
                offspring from the highest scoring individual
                with mutations
        """
        mc_individual,mc_maxscore=get_super_performer(mc_gen,scores)
        mc_newgen=[mc_gen[mc_individual].copy() for i in range(population)]

        for i in range(len(mc_newgen)):

                for j in range(len(mc_newgen[i])):

                        mc_r=randint(0,100)

                        if mutation_chance>mc_r:

                                mc_newgen[i][j]=1-mc_newgen[i][j]

        return mc_newgen,mc_individual,mc_maxscore

def main():
        weight=[7,2,1,9,11,2,3,4,7,9,10,5,4,2,5,6]
        value=[5,4,7,2,5,4,7,8,9,10,2,6,4,1,8,9]
        maxweight=15
        gen=[[randint(0,1) for i in range(len(weight))] for j in range(population)]
        for i in range(70):
                scores=getScores(weight,value,maxweight,gen)
                gen,individual,maximumscore=mutant_clone(gen,scores)
                print(f"gen#{i+1}",gen[individual],maximumscore)
main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# File: data.py

import cv2
import numpy as np
import copy
import random
import math
from utils.np_box_ops import iou as np_iou
from tensorpack.utils.argtools import memoized, log_once
from utils.np_box_ops import area as np_area
from tensorpack.dataflow import (
    MapData, imgaug, AugmentImageComponent, TestDataSpeed, MultiProcessMapData,
    MapDataComponent, DataFromList, PrefetchDataZMQ, BatchData)
import tensorpack.utils.viz as tpviz

#from pycocotools.coco import COCO
#from pycocotools.cocoeval import COCOeval
#import pycocotools.mask as cocomask

from airbus import Detection, ResnetDetection

from utils.generate_anchors import generate_anchors
from utils.box_ops import get_iou_callable
from common import (
    DataFromListOfDict, CustomResize,
    box_to_point8, point8_to_box, segmentation_to_mask)
import config
from imaug import (do_flip_transpose2, get_resnet_augmentor)
from albumentations import (
    HorizontalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine,
    IAASharpen, IAAEmboss, RandomContrast, RandomBrightness, Flip, OneOf, Compose, RandomGamma
)

class MalformedData(BaseException):
    pass

@memoized
def get_all_anchors_FPN(stride, sizes):
    # Generates a NAx4 matrix of anchor boxes in (x1, y1, x2, y2) format. Anchors
    # are centered on stride / 2, have (approximate) sqrt areas of the specified
    # sizes, and aspect ratios as given.
    cell_anchors = generate_anchors(
        stride,
        scales=np.array([sizes], dtype=np.float) / stride,
        ratios=np.array(config.ANCHOR_RATIOS, dtype=np.float))
    # anchors are intbox here.
    # anchors at featuremap [0,0] are centered at fpcoor (8,8) (half of stride)
    fpn_max_size = 32 * np.ceil(
        config.MAX_SIZE / 32
    )
    field_size = int(np.ceil(fpn_max_size / float(stride)))

    # field_size = config.MAX_SIZE // stride
    shifts = np.arange(0, field_size) * stride
    shift_x, shift_y = np.meshgrid(shifts, shifts)
    shift_x = shift_x.flatten()
    shift_y = shift_y.flatten()
    shifts = np.vstack((shift_x, shift_y, shift_x, shift_y)).transpose()
    # Kx4, K = field_size * field_size
    K = shifts.shape[0]

    A = cell_anchors.shape[0]
    field_of_anchors = (
        cell_anchors.reshape((1, A, 4)) +
        shifts.reshape((1, K, 4)).transpose((1, 0, 2)))
    field_of_anchors = field_of_anchors.reshape((field_size, field_size, A, 4))
    # FSxFSxAx4
    assert np.all(field_of_anchors == field_of_anchors.astype('int32'))
    field_of_anchors = field_of_anchors.astype('float32')
    field_of_anchors[:, :, :, [2, 3]] += 1
    return field_of_anchors

@memoized
def get_all_anchors(
        stride=config.ANCHOR_STRIDE,
        sizes=config.ANCHOR_SIZES,
        ratios=config.ANCHOR_RATIOS):
    """
    Get all anchors in the largest possible image, shifted, floatbox

    Returns:
        anchors: SxSxNUM_ANCHORx4, where S == MAX_SIZE//STRIDE, floatbox
        The layout in the NUM_ANCHOR dim is NUM_RATIO x NUM_SCALE.

    """
    # Generates a NAx4 matrix of anchor boxes in (x1, y1, x2, y2) format. Anchors
    # are centered on stride / 2, have (approximate) sqrt areas of the specified
    # sizes, and aspect ratios as given.
    cell_anchors = generate_anchors(
        stride,
        scales=np.array(sizes, dtype=np.float) / stride,
        ratios=np.array(ratios, dtype=np.float))
    # anchors are intbox here.
    # anchors at featuremap [0,0] are centered at fpcoor (8,8) (half of stride)

    field_size = config.MAX_SIZE // stride
    shifts = np.arange(0, field_size) * stride
    shift_x, shift_y = np.meshgrid(shifts, shifts)
    shift_x = shift_x.flatten()
    shift_y = shift_y.flatten()
    shifts = np.vstack((shift_x, shift_y, shift_x, shift_y)).transpose()
    # Kx4, K = field_size * field_size
    K = shifts.shape[0]

    A = cell_anchors.shape[0]
    field_of_anchors = (
        cell_anchors.reshape((1, A, 4)) +
        shifts.reshape((1, K, 4)).transpose((1, 0, 2)))
    field_of_anchors = field_of_anchors.reshape((field_size, field_size, A, 4))
    # FSxFSxAx4
    #assert np.all(field_of_anchors == field_of_anchors.astype('int32'))
    field_of_anchors = field_of_anchors.astype('float32')
    field_of_anchors[:, :, :, [2, 3]] += 1
    return field_of_anchors

def get_anchor_labels(anchors, gt_boxes, crowd_boxes):
    """
    Label each anchor as fg/bg/ignore.
    Args:
        anchors: Ax4 float
        gt_boxes: Bx4 float
        crowd_boxes: Cx4 float

    Returns:
        anchor_labels: (A,) int. Each element is {-1, 0, 1}
        anchor_boxes: Ax4. Contains the target gt_box for each anchor when the anchor is fg.
    """
    # This function will modify labels and return the filtered inds
    def filter_box_label(labels, value, max_num):
        curr_inds = np.where(labels == value)[0]
        if len(curr_inds) > max_num:
            disable_inds = np.random.choice(
                curr_inds, size=(len(curr_inds) - max_num),
                replace=False)
            labels[disable_inds] = -1    # ignore them
            curr_inds = np.where(labels == value)[0]
        return curr_inds

    #bbox_iou_float = get_iou_callable()

    NA, NB = len(anchors), len(gt_boxes)
    assert NB > 0  # empty images should have been filtered already
    #box_ious = bbox_iou_float(anchors, gt_boxes)  # NA x NB
    box_ious = np_iou(anchors, gt_boxes)  # NA x NB

    ious_argmax_per_anchor = box_ious.argmax(axis=1)  # NA,
    ious_max_per_anchor = box_ious.max(axis=1)
    ious_max_per_gt = np.amax(box_ious, axis=0, keepdims=True)  # 1xNB
    # for each gt, find all those anchors (including ties) that has the max ious with it
    anchors_with_max_iou_per_gt = np.where(box_ious == ious_max_per_gt)[0]

    # Setting NA labels: 1--fg 0--bg -1--ignore
    anchor_labels = -np.ones((NA,), dtype='int32')   # NA,

    # the order of setting neg/pos labels matter
    anchor_labels[anchors_with_max_iou_per_gt] = 1
    anchor_labels[ious_max_per_anchor >= config.POSITIVE_ANCHOR_THRES] = 1
    anchor_labels[ious_max_per_anchor < config.NEGATIVE_ANCHOR_THRES] = 0

    # First label all non-ignore candidate boxes which overlap crowd as ignore
    #if crowd_boxes.size > 0:
    #    cand_inds = np.where(anchor_labels >= 0)[0]
    #    cand_anchors = anchors[cand_inds]
    #    ious = bbox_iou_float(cand_anchors, crowd_boxes)
    #    overlap_with_crowd = cand_inds[ious.max(axis=1) > config.CROWD_OVERLAP_THRES]
    #    anchor_labels[overlap_with_crowd] = -1

    # Filter fg labels: ignore some fg if fg is too many
    target_num_fg = int(config.RPN_BATCH_PER_IM * config.RPN_FG_RATIO)
    fg_inds = filter_box_label(anchor_labels, 1, target_num_fg)
    # Note that fg could be fewer than the target ratio

    # filter bg labels. num_bg is not allowed to be too many
    old_num_bg = np.sum(anchor_labels == 0)
    if old_num_bg == 0 or len(fg_inds) == 0:
        # No valid bg/fg in this image, skip.
        # This can happen if, e.g. the image has large crowd.
        raise MalformedData("No valid foreground/background for RPN!")
    target_num_bg = config.RPN_BATCH_PER_IM - len(fg_inds)
    filter_box_label(anchor_labels, 0, target_num_bg)   # ignore return values

    # Set anchor boxes: the best gt_box for each fg anchor
    anchor_boxes = np.zeros((NA, 4), dtype='float32')
    fg_boxes = gt_boxes[ious_argmax_per_anchor[fg_inds], :]
    anchor_boxes[fg_inds, :] = fg_boxes
    return anchor_labels, anchor_boxes

def get_rpn_anchor_input_FPN(im, boxes, is_crowd):
    def clip_boxes(boxes, shape):
        orig_shape = boxes.shape
        boxes = boxes.reshape([-1, 4])
        h, w = shape
        boxes[:, [0, 1]] = np.maximum(boxes[:, [0, 1]], 0)
        boxes[:, 2] = np.minimum(boxes[:, 2], w)
        boxes[:, 3] = np.minimum(boxes[:, 3], h)
        return boxes.reshape(orig_shape)

    def filter_box_inside(im, boxes):
        h, w = im.shape[:2]
        indices = np.where(
            (boxes[:, 0] >= 0) &
            (boxes[:, 1] >= 0) &
            (boxes[:, 2] <= w) &
            (boxes[:, 3] <= h))[0]
        return indices

    boxes = boxes.copy()
    fpn_boxes = []
    fpn_labels = []
    crowd_boxes = boxes[is_crowd == 1]
    non_crowd_boxes = boxes[is_crowd == 0]
    for stride, size in zip(config.FPN_STRIDES, config.FPN_SIZES):
        ALL_ANCHORS = get_all_anchors_FPN(stride, size)
        H, W = im.shape[:2]
        featureH, featureW = H // stride, W // stride
        # fHxfWxAx4
        featuremap_anchors = ALL_ANCHORS[:featureH, :featureW, :, :]
        featuremap_anchors_flatten = featuremap_anchors.reshape((-1, 4))
        """ no clip """
        featuremap_anchors_flatten = clip_boxes(featuremap_anchors_flatten, im.shape[:2])
        """ no clip """
        fpn_boxes.append(featuremap_anchors_flatten)
        #print("featuremap_{}".format(stride), featuremap_anchors_flatten.shape)
    all_featuremap_anchors_flatten = np.vstack(fpn_boxes)
    #print("all: ", all_featuremap_anchors_flatten.shape)
    # anchor of all featuremaps
    inside_ind = filter_box_inside(im, all_featuremap_anchors_flatten)
    inside_anchors = all_featuremap_anchors_flatten[inside_ind, :]
    anchor_labels, anchor_boxes = get_anchor_labels(inside_anchors, non_crowd_boxes, crowd_boxes)
    """
    featuremap_labels = -np.ones((featureH * featureW * config.NUM_ANCHOR, ), dtype='int32')
    featuremap_labels[inside_ind] = anchor_labels
    # featuremap_labels = featuremap_labels.reshape((featureH, featureW, config.NUM_ANCHOR))
    featuremap_boxes = np.zeros((featureH * featureW * config.NUM_ANCHOR, 4), dtype='float32')
    featuremap_boxes[inside_ind, :] = anchor_boxes
    # featuremap_boxes = featuremap_boxes.reshape((featureH, featureW, config.NUM_ANCHOR, 4))
    return featuremap_labels, featuremap_boxes
    """
    return anchor_labels, anchor_boxes

def get_rpn_anchor_input(im, boxes, is_crowd):
    """
    Args:
        im: an image
        boxes: nx4, floatbox, gt. shoudn't be changed
        is_crowd: n,

    Returns:
        The anchor labels and target boxes for each pixel in the featuremap.
        fm_labels: fHxfWxNA
        fm_boxes: fHxfWxNAx4
    """
    boxes = boxes.copy()

    ALL_ANCHORS = get_all_anchors()
    H, W = im.shape[:2]
    featureH, featureW = H // config.ANCHOR_STRIDE, W // config.ANCHOR_STRIDE

    def clip_boxes(boxes, shape):
        """
        Args:
            boxes: (...)x4, float
            shape: h, w
        """
        orig_shape = boxes.shape
        boxes = boxes.reshape([-1, 4])
        h, w = shape
        boxes[:, [0, 1]] = np.maximum(boxes[:, [0, 1]], 0)
        boxes[:, 2] = np.minimum(boxes[:, 2], w)
        boxes[:, 3] = np.minimum(boxes[:, 3], h)
        return boxes.reshape(orig_shape)

    def filter_box_inside(im, boxes):
        h, w = im.shape[:2]
        indices = np.where(
            (boxes[:, 0] >= 0) &
            (boxes[:, 1] >= 0) &
            (boxes[:, 2] <= w) &
            (boxes[:, 3] <= h))[0]
        return indices

    crowd_boxes = boxes[is_crowd == 1]
    non_crowd_boxes = boxes[is_crowd == 0]

    # fHxfWxAx4
    featuremap_anchors = ALL_ANCHORS[:featureH, :featureW, :, :]
    featuremap_anchors_flatten = featuremap_anchors.reshape((-1, 4))
    # only use anchors inside the image
    ### clip outside box and use them
    featuremap_anchors_flatten = clip_boxes(featuremap_anchors_flatten, im.shape[:2])
    ###
    inside_ind = filter_box_inside(im, featuremap_anchors_flatten)
    inside_anchors = featuremap_anchors_flatten[inside_ind, :]

    anchor_labels, anchor_boxes = get_anchor_labels(inside_anchors, non_crowd_boxes, crowd_boxes)

    # Fill them back to original size: fHxfWx1, fHxfWx4
    featuremap_labels = -np.ones((featureH * featureW * config.NUM_ANCHOR, ), dtype='int32')
    featuremap_labels[inside_ind] = anchor_labels
    featuremap_labels = featuremap_labels.reshape((featureH, featureW, config.NUM_ANCHOR))
    featuremap_boxes = np.zeros((featureH * featureW * config.NUM_ANCHOR, 4), dtype='float32')
    featuremap_boxes[inside_ind, :] = anchor_boxes
    featuremap_boxes = featuremap_boxes.reshape((featureH, featureW, config.NUM_ANCHOR, 4))
    return featuremap_labels, featuremap_boxes

def multi_mask_to_annotation(multi_mask):
    H,W      = multi_mask.shape[:2]
    box      = []
    label    = []
    instance = []
    is_crowd = []
    raw_label = np.unique(multi_mask)
    for i in raw_label:
        _la = i // 1000
        _ins_a = i % 1000
        if i == 0:
            # ignore background
            continue
        mask = (multi_mask==i)
        if mask.sum()>1:
            y,x = np.where(mask)
            y0 = y.min()
            y1 = y.max()
            x0 = x.min()
            x1 = x.max()
            w = (x1-x0)+1
            h = (y1-y0)+1

            border = max(1, round(0.1*min(w,h)))
            x0 = x0-border
            x1 = x1+border
            y0 = y0-border
            y1 = y1+border

            #clip
            x0 = max(1,x0)
            y0 = max(1,y0)
            x1 = min(W-1,x1)
            y1 = min(H-1,y1)

            box.append([x0,y0,x1,y1])
            label.append(_la)
            instance.append(mask)
            is_crowd.append(0) # ignored cases

    box      = np.array(box, np.float32)
    label    = np.array(label, np.int32)
    instance = np.array(instance, np.float32)
    is_crowd = np.array(is_crowd, np.int32)

    return box, label, instance, is_crowd

def better_resize(img, size, max_size=config.MAX_SIZE):
    h, w = img.shape[:2]
    scale = size * 1.0 / min(h, w)
    if h < w:
        newh, neww = size, scale * w
    else:
        newh, neww = scale * h, size
    if max(newh, neww) > max_size:
        scale = max_size * 1.0 / max(newh, neww)
        newh = newh * scale
        neww = neww * scale
    neww = int(neww + 0.5)
    newh = int(newh + 0.5)
    return newh, neww

def fix_resize_transform_range(image, semantic_label, sizes, u=0.5):
    H,W = image.shape[:2]
    #s = np.random.choice(sizes)
    s = random.randint(sizes[0], sizes[1])
    #s = sizes
    h, w = better_resize(image, s)
    if (H,W) != (h,w):
        image = cv2.resize(image, (w,h))

        semantic_label = cv2.resize(semantic_label, (w, h), interpolation=cv2.INTER_NEAREST)
        semantic_label = semantic_label.astype(np.int32)
    return image, semantic_label

def fix_resize_transform_scale(image, semantic_label, sizes, u=0.5):
    H,W = image.shape[:2]
    s = np.random.choice(sizes)
    #s = sizes
    h, w = better_resize(image, s)
    if (H,W) != (h,w):
        image = cv2.resize(image, (w,h))

        semantic_label = cv2.resize(semantic_label, (w, h), interpolation=cv2.INTER_NEAREST)
        semantic_label = semantic_label.astype(np.int32)
    return image, semantic_label

def random_crop_box_center(im, label, crop_height=800, crop_width=1024):
    crop_height = min(crop_height, im.shape[0])
    crop_width = min(crop_width, im.shape[1])

    instanceId = np.unique(label[label!=0])
    if len(instanceId) == 0:
        return im, label
    choiced_id = np.random.choice(instanceId)
    mask_true = np.where(label==choiced_id)
    seed = np.random.choice(range(len(mask_true[0])))
    coordinate = (mask_true[0][seed], mask_true[1][seed])
    ### get crop window ###
    start_y = max(0, coordinate[0] - crop_height//2)
    end_y = min(im.shape[0], coordinate[0] + crop_height//2)
    start_x = max(0, coordinate[1] - crop_width//2)
    end_x = min(im.shape[1], coordinate[1] + crop_width//2)
    ### random shift ###
    shift_limit = (-0.2, 0.2)
    dx = round(random.uniform(shift_limit[0],shift_limit[1])*crop_width )
    dy = round(random.uniform(shift_limit[0],shift_limit[1])*crop_height)
    start_x = max(start_x + dx, 0)
    end_x = min(end_x + dx, im.shape[1])
    start_y = max(start_y + dy, 0)
    end_y = min(end_y + dy, im.shape[0])
    ###
    #cropped_im = im[start_y:end_y, start_x:end_x, :]
    return im[start_y:end_y, start_x:end_x, :],  label[start_y:end_y, start_x:end_x]

def getAnnotation(df, imageId):
    def rle_decode(mask_rle, shape=(768, 768)):
        s = mask_rle.split()
        starts =  np.asarray(s[0::2], dtype=int)
        lengths = np.asarray(s[1::2], dtype=int)

        starts -= 1
        ends = starts + lengths
        img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
        for lo, hi in zip(starts, ends):
            img[lo:hi] = 1
        return img.reshape(shape).T  # Needed to align to RLE direction

    def instances_to_multimask(instance):
        H,W = instance.shape[1:3]
        multi_mask = np.zeros((H,W),np.int32)
        num_masks = len(instance)
        for i in range(num_masks):
            multi_mask[instance[i]>0] = 1000 + i

        return multi_mask
    try:
        rle_masks = df.loc[imageId, 'EncodedPixels']
    except:
        return None
    if isinstance(rle_masks, str):
        rle_masks = [rle_masks]
    else:
        rle_masks = rle_masks.tolist()
    rle_masks = [rle_decode(m) for m in rle_masks]
    multimask = instances_to_multimask(np.array(rle_masks))
    return multimask

def strong_aug(p=.5):
    return Compose([
        RandomRotate90(),
        Flip(),
        Transpose(),
        OneOf([
            IAAAdditiveGaussianNoise(),
            GaussNoise(),
        ], p=0.2),
        OneOf([
            MotionBlur(p=.2),
            MedianBlur(blur_limit=3, p=.1),
            Blur(blur_limit=3, p=.1),
        ], p=0.2),
        ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=.2),
        OneOf([
            OpticalDistortion(p=0.3),
            GridDistortion(p=.1),
            IAAPiecewiseAffine(p=0.3),
        ], p=0.2),
        OneOf([
            CLAHE(clip_limit=2),
            IAASharpen(),
            IAAEmboss(),
        ], p=0.3),
        HueSaturationValue(p=0.3),
        RandomBrightness(p=0.2,limit=0.2),
        RandomContrast(p=0.2,limit=0.2),
        RandomGamma(p=0.2)
    ], p=p)

def pad_to_factor(image, factor=32):
    height,width = image.shape[:2]
    h = math.ceil(height/factor)*factor
    w = math.ceil(width/factor)*factor

    image = cv2.copyMakeBorder(image, top=0, bottom=h-height, left=0, right=w-width,
                               borderType= cv2.BORDER_REFLECT101, value=[0,0,0] )

    return image

def get_train_dataflow(add_mask=True):
    """
    Return a training dataflow. Each datapoint is:
    image, fm_labels, fm_boxes, gt_boxes, gt_class [, masks]
    """
    imgs = Detection.load_many(
        config.BASEDIR, config.TRAIN_DATASET, add_gt=True, add_mask=add_mask)
    # Valid training images should have at least one fg box.
    # But this filter shall not be applied for testing.
    imgs = list(imgs)
    import os
    import pandas as pd
    csv_path = os.path.join(config.BASEDIR, 'train_ship_segmentations_v2.csv')
    if config.PSEUDO:
        pseudo_csv_path = './pseudo_label1.csv'
        df = pd.concat((pd.read_csv(f, engine="python") for f in [csv_path, pseudo_csv_path]))
        df = df.dropna(axis=0)
        df = df.set_index('ImageId')
    else:
        df = pd.read_csv(csv_path, engine="python")
        df = df.dropna(axis=0)
        df = df.set_index('ImageId')
    ds = DataFromList(imgs, shuffle=True)
    def preprocess(img):
        im, fname = img['image_data'], img['id']
        multi_mask = getAnnotation(df, fname)
        if multi_mask is None:
            return None
        im = cv2.imread(im)
        #============================
        #if random.random() > 0.5:
        #    im = np.fliplr(im) # h, w, 3
        #    multi_mask = np.fliplr(multi_mask)
        #im, multi_mask = do_flip_transpose2(im, multi_mask, type=random.randint(0,7))
        augmented = strong_aug()(image=im, mask=multi_mask)
        im, multi_mask = augmented['image'], augmented['mask']
        #============================
        # Resize
        im, multi_mask = fix_resize_transform_range(im, multi_mask, [768, 2000], 1.0)
        im = pad_to_factor(im)
        multi_mask = pad_to_factor(multi_mask)

        boxes, klass, masks, is_crowd = multi_mask_to_annotation(multi_mask)
        if len(boxes) == 0 or np.min(np_area(boxes)) <= 0:
            log_once("Input have zero area box: {}".format(fname), 'warn')
            return None
        # rpn anchor:
        try:
            if config.FPN:
                fm_labels, fm_boxes = get_rpn_anchor_input_FPN(im, boxes, is_crowd)
            else:
                fm_labels, fm_boxes = get_rpn_anchor_input(im, boxes, is_crowd)

            if not len(boxes):
                raise MalformedData("No valid gt_boxes!")
        except MalformedData as e:
            log_once("Input {} is filtered for training: {}".format(fname, str(e)), 'warn')
            return None

        ret = [im, fm_labels, fm_boxes, boxes, klass, masks]
        """
        from viz import draw_annotation, draw_mask
        viz = draw_annotation(im, boxes, klass)
        for ind, mask in enumerate(masks):
            viz = draw_mask(viz, mask)
            cv2.imwrite("./test_{}.jpg".format(np.random.rand()), viz)
        if (len(boxes) > 3):
            exit()
        """
        return ret

    ds = MapData(ds, preprocess)
    ds = PrefetchDataZMQ(ds, 6)
    return ds

def get_test_dataflow(add_mask=True):
    """
    Return a training dataflow. Each datapoint is:
    image, fm_labels, fm_boxes, gt_boxes, gt_class [, masks]
    """
    imgs = Detection.load_many(
        config.BASEDIR, config.VAL_DATASET, add_gt=False, add_mask=add_mask)
    # no filter for training
    ds = DataFromListOfDict(imgs, ['image_data', 'id'])

    def f(image):
        im = cv2.imread(image)
        return im

    ds = MapDataComponent(ds, f, 0)
    ds = PrefetchDataZMQ(ds, 1)
    return ds

def get_resnet_train_dataflow():
    imgs = ResnetDetection.load_many(
        config.BASEDIR, config.TRAIN_DATASET)
    # Valid training images should have at least one fg box.
    # But this filter shall not be applied for testing.
    imgs = list(imgs)

    ds = DataFromList(imgs, shuffle=True)
    augmentors = get_resnet_augmentor()
    def preprocess(img):
        im, fname, label = img['image_data'], img['id'], img['with_ship']
        im = cv2.imread(im)
        #============Aug================
        im = cv2.resize(im, (config.RESNET_SIZE, config.RESNET_SIZE))
        augmented = strong_aug()(image=im)
        im = augmented['image']
        # im, multi_mask = do_flip_transpose2(im, multi_mask, type=random.randint(0,7))
        #============================
        ret = [im, label]
        return ret
    ds = MapData(ds, preprocess)
    ds = AugmentImageComponent(ds, augmentors, copy=False)
    ds = BatchData(ds, config.RESNET_BATCH)
    ds = PrefetchDataZMQ(ds, 6)
    return ds

def get_resnet_val_dataflow():
    imgs = ResnetDetection.load_many(
        config.BASEDIR, config.VAL_DATASET)
    imgs = list(imgs)
    # ds = DataFromListOfDict(imgs, ['image_data', 'with_ship', 'id'])
    ds = DataFromList(imgs, shuffle=False)
    def f(img):
        image, label = img['image_data'], img['with_ship']
        im = cv2.imread(image)
        im = cv2.resize(im, (config.RESNET_SIZE, config.RESNET_SIZE))
        return [im, label]

    ds = MapData(ds, f)
    ds = BatchData(ds, config.RESNET_BATCH)
    ds = PrefetchDataZMQ(ds, 1)
    return ds

#=====================================#

def get_debug_dataflow(add_mask=True, imageHW=768):
    """
    Return a training dataflow. Each datapoint is:
    image, fm_labels, fm_boxes, gt_boxes, gt_class [, masks]
    """
    imgs = Detection.load_many(
        config.BASEDIR, config.TRAIN_DATASET, add_gt=True, add_mask=add_mask)
    # Valid training images should have at least one fg box.
    # But this filter shall not be applied for testing.
    imgs = list(imgs)
    import os
    import pandas as pd
    csv_path = os.path.join(config.BASEDIR, 'train_ship_segmentations_v2.csv')
    df = pd.read_csv(csv_path, engine="python")
    df = df.dropna(axis=0)
    df = df.set_index('ImageId')

    ds = DataFromList(imgs, shuffle=True)
    def preprocess(img):
        im, fname = img['image_data'], img['id']
        multi_mask = getAnnotation(df, fname)
        im = cv2.imread(im)
        im, multi_mask = fix_resize_transform_range(im, multi_mask, [imageHW, imageHW], 1.0)
        boxes, klass, masks, is_crowd = multi_mask_to_annotation(multi_mask)
        return boxes
    ds = MapData(ds, preprocess)
    ds = PrefetchDataZMQ(ds, 6)
    return ds

def iou(box, clusters):
    """
    Calculates the Intersection over Union (IoU) between a box and k clusters.
    :param box: tuple or array, shifted to the origin (i. e. width and height)
    :param clusters: numpy array of shape (k, 2) where k is the number of clusters
    :return: numpy array of shape (k, 0) where k is the number of clusters
    """
    x = np.minimum(clusters[:, 0], box[0])
    y = np.minimum(clusters[:, 1], box[1])
    if np.count_nonzero(x == 0) > 0 or np.count_nonzero(y == 0) > 0:
        raise ValueError("Box has no area")

    intersection = x * y
    box_area = box[0] * box[1]
    cluster_area = clusters[:, 0] * clusters[:, 1]

    iou_ = intersection / (box_area + cluster_area - intersection)

    return iou_

def avg_iou(boxes, clusters):
    """
    Calculates the average Intersection over Union (IoU) between a numpy array of boxes and k clusters.
    :param boxes: numpy array of shape (r, 2), where r is the number of rows
    :param clusters: numpy array of shape (k, 2) where k is the number of clusters
    :return: average IoU as a single float
    """
    return np.mean([np.max(iou(boxes[i], clusters)) for i in range(boxes.shape[0])])

def translate_boxes(boxes):
    """
    Translates all the boxes to the origin.
    :param boxes: numpy array of shape (r, 4)
    :return: numpy array of shape (r, 2)
    """
    new_boxes = boxes.copy()
    for row in range(new_boxes.shape[0]):
        new_boxes[row][2] = np.abs(new_boxes[row][2] - new_boxes[row][0])
        new_boxes[row][3] = np.abs(new_boxes[row][3] - new_boxes[row][1])
    return np.delete(new_boxes, [0, 1], axis=1)

def kmeans(boxes, k, dist=np.median):
    """
    Calculates k-means clustering with the Intersection over Union (IoU) metric.
    :param boxes: numpy array of shape (r, 2), where r is the number of rows
    :param k: number of clusters
    :param dist: distance function
    :return: numpy array of shape (k, 2)
    """
    rows = boxes.shape[0]

    distances = np.empty((rows, k))
    last_clusters = np.zeros((rows,))

    np.random.seed()

    # the Forgy method will fail if the whole array contains the same rows
    clusters = boxes[np.random.choice(rows, k, replace=False)]

    while True:
        for row in range(rows):
            distances[row] = 1 - iou(boxes[row], clusters)

        nearest_clusters = np.argmin(distances, axis=1)

        if (last_clusters == nearest_clusters).all():
            break

        for cluster in range(k):
            clusters[cluster] = dist(boxes[nearest_clusters == cluster], axis=0)

        last_clusters = nearest_clusters

    return clusters

def cal_avg_iou_per_gt(gt_boxes, sizes, ratios, imageHW=768):
    def filter_box_inside(im, boxes):
        h, w = im.shape[:2]
        indices = np.where(
            (boxes[:, 0] >= 0) &
            (boxes[:, 1] >= 0) &
            (boxes[:, 2] <= w) &
            (boxes[:, 3] <= h))[0]
        return indices
    im = np.zeros((imageHW, imageHW))
    ALL_ANCHORS = get_all_anchors(sizes=sizes, ratios=ratios)
    H, W = im.shape[:2]
    featureH, featureW = H // 16, W // 16
    featuremap_anchors = ALL_ANCHORS[:featureH, :featureW, :, :]
    featuremap_anchors_flatten = featuremap_anchors.reshape((-1, 4))
    inside_ind = filter_box_inside(im, featuremap_anchors_flatten)
    inside_anchors = featuremap_anchors_flatten[inside_ind, :]

    anchors = inside_anchors
    box_ious = np_iou(anchors, gt_boxes)  # NA x NB
    ious_argmax_per_anchor = box_ious.argmax(axis=1)  # NA,
    ious_max_per_anchor = box_ious.max(axis=1)
    ious_max_per_gt = np.amax(box_ious, axis=0, keepdims=True)  # 1xNB
    return ious_max_per_gt.mean()

if __name__ == '__main__':

    imgs = Detection.load_many(
        config.BASEDIR, config.TRAIN_DATASET, add_gt=True, add_mask=True)
    # Valid training images should have at least one fg box.
    # But this filter shall not be applied for testing.
    imgs = list(imgs)
    import os
    import pandas as pd
    csv_path = os.path.join(config.BASEDIR, 'train_ship_segmentations_v2.csv')
    df = pd.read_csv(csv_path, engine="python")
    df = df.dropna(axis=0)
    df = df.set_index('ImageId')
    from tqdm import tqdm
    for img in tqdm(imgs, total=len(imgs)):
        im, fname = img['image_data'], img['id']
        multi_mask = getAnnotation(df, fname)

        im = cv2.imread(im)
        #============================
        # Resize
        augmented = strong_aug()(image=im, mask=multi_mask)
        im, multi_mask = augmented['image'], augmented['mask']
        boxes, klass, masks, is_crowd = multi_mask_to_annotation(multi_mask)
        if len(boxes) == 0 or np.min(np_area(boxes)) <= 0:
            log_once("Input have zero area box: {}".format(fname), 'warn')
            print(boxes)
            exit()
        """
        from viz import draw_annotation, draw_mask
        viz = draw_annotation(im, boxes, klass)
        for ind, mask in enumerate(masks):
            viz = draw_mask(viz, mask)
            cv2.imwrite("./eval_gt/{}.jpg".format(fname), viz)
        """
    """
    # for each gt, find all those anchors (including ties) that has the max ious with it
    ANCHOR_SIZES = (32,64,128,256,512)
    RAIOS = (0.5,1,2)
    #ANCHOR_SIZES = (16, 32, 64, 128, 256)
    from tensorpack.dataflow import PrintData
    from tqdm import tqdm
    imageHW = 2000
    ds = get_debug_dataflow(imageHW=imageHW)
    ds.reset_state()
    all_boxes = []
    all_boxes_hw = []
    all_boxes_ratio = []
    for idx, boxes in tqdm(enumerate(ds.get_data()), total=ds.size()):
        for i in boxes:
            all_boxes.append(i)
            all_boxes_hw.append([i[2]-i[0], i[3]-i[1]])
            all_boxes_ratio.append( (i[2]-i[0])/float(i[3]-i[1]) )
    print(np.array(all_boxes_ratio).mean())
    all_boxes = np.array(all_boxes)
    all_boxes_hw = np.array(all_boxes_hw)
    print(cal_avg_iou_per_gt(all_boxes, sizes=ANCHOR_SIZES, ratios=RAIOS, imageHW=imageHW))
    out = kmeans(all_boxes_hw, k=5)
    print("Accuracy: {:.2f}%".format(avg_iou(all_boxes_hw, out) * 100))
    print("Boxes:\n {}".format(out))
    """


#Problem Link:- https://leetcode.com/problems/construct-binary-search-tree-from-preorder-traversal/

"""Return the root node of a binary search tree that matches the given
preorder traversal."""

# Definition for a binary tree node.
class TreeNode:
    def __init__(self, val=0, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right

class Solution:
    def bstFromPreorder(self, preorder):
        if len(preorder) == 0:
            return
        inorder = sorted(preorder)
        return self.helper(0,0,len(inorder)-1,inorder,preorder)

    def helper(self,preStart,inStart,inEnd,inorder,preorder):
        if (preStart > len(preorder)-1) or (inStart > inEnd):
            return
        root = TreeNode(preorder[preStart])
        for i in range(len(inorder)):
            if root.val == inorder[i]:
                index = i
                break
        root.left = self.helper(preStart+1,inStart,index-1,inorder,preorder)
        root.right = self.helper(preStart+index-inStart+1,index+1,inEnd,inorder,preorder)
        return root


from ml_core.src.main.code.pipeline.transformers.feature_selector import FeatureSelector
import pandas as pd
import numpy as np

class TestFeatureSelector:
    def setup(self):
        self.feature_selector = FeatureSelector(all_features = ['nanny', 'buddy', 'lava', 'whiskey'])

    def test_we_select_features(self):
        nanny_frame = pd.DataFrame({
            'nanny': [5, 1, 1, 2, 3, 5],
            'buddy': [1,2,3,4,5,6],
            'lava': [1,5,4,3,2,1],
            'whiskey': [np.nan] * 6
        })
        labels = np.array([1,2,3,4,5,6])
        df, _ = self.feature_selector.transform(nanny_frame, labels)
        assert len(list(df.columns)) == 3


import os
import common
import string
import MyThread
from time import sleep
import requests
from urllib import request
import urllib

import socket  #
socket.setdefaulttimeout(10.0)   #设置全局的socket超时
import ssl
ssl._create_default_https_context = ssl._create_unverified_context

def downHtml(http,encodeStr="utf-8"):
    '''
    下载网址 ，获得HTML内容
    :param http:      网址
    :param encodeStr: 网址编码
    :return html:     返回获得的网址内容
    '''
    html = ""
    while html=="":
        try:
            re= requests.get(http,timeout=3)
            re.encoding = encodeStr    #直接从原网页获得编码信息  或者# re.encoding = re.apparent_encoding 这种方式自动分析，占用时间可能长
            html=re.text
        except:                          #请求不成功的话，循环 但sleep一段时间，保持频率不要太高，以免被拒绝
            sleep(5)
            continue
    return html

def bExist(httpPath):

    bExist=True
    while bExist:
        try:
           request.urlopen(httpPath, timeout=10)
           break
        except urllib.error.HTTPError as e:
            if e.code==404:
                bExist = False
        except urllib.error.URLError as e:
            print(e.reason)
        except socket.timeout as e:
            print(e)
    return bExist

    # request.urlretrieve(httpPath,localPath)  直接下载 出现10060错误
    # requests.get('https://www.zhihu.com/',verify=False)  verify=False 要不然会有SSL错误
def downImg(httpPath,localPath):
    html = ""
    num=0
    nImgType=0 #先尝试PNG，再尝试JPG
    while html=="":
         try:
             num+=1
             if num>1:
                print("--第%d次尝试--" %num)

             headers={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36"}

             req=request.Request(url=httpPath, headers=headers)
             #resp = request.urlopen(httpPath,timeout=60,headers=headers)
             resp = request.urlopen(req, timeout=60)
             html = resp.read()

         except urllib.error.HTTPError as e:
             if e.code==404:
                nImgType+=1
                httpPath='.'.join(httpPath.split(".")[0:-1])+".jpg"
                localPath=localPath.split(".")[0]+".jpg"
                if nImgType>=2:
                    return False #不存在
                else:
                    continue
         except (urllib.error.URLError, socket.timeout,Exception)as e:
             print(e)
             if num>50:
                 return False
             sleep(5)
             continue

    picFile = open(localPath, "wb")
    picFile.write(html)
    picFile.close()
    return True

#下载图片
def DownImg(domain,subDirList,subsubImgDir):
    '''
    :param subDirList:    文件夹列表
    :param subsubImgDir:  新创建的子文件夹，用以保存图像
    :return:
    '''
    for dir in subDirList:
        print("-------%s------" %dir)
        imgList=[]
        common.readLinkFile(dir+"\\img.txt",imgList)

        if not os.path.exists(dir + "\\%s\\"%subsubImgDir):
            os.makedirs(dir + "\\%s\\"%subsubImgDir)
        for img in imgList:
            imgName=img.split("/")[-1]
            print(imgName)
            common.downImg(domain+img,dir+"\\%s\\%s" %(subsubImgDir,imgName))

def GetRealImg(oriImg,ext=".png"):
    # img/flower2/s-flower5526.jpg
    # img/flower/flower5526.png
    splitlist = oriImg.split("/")
    s1=splitlist[0]                        # img
    s2=splitlist[1].rstrip(string.digits)  # flower
    s3=splitlist[2]                        # s-flower5526.jpg
    s3list=s3.split(".")
    s3=s3list[0]                           # s-flower5526
    subSplitList=s3.split("-")
    stemp="-".join(subSplitList[1:])        #flower5526
    return s1+"/"+s2+"/"+stemp+ext

#将图片链接，重新解析为新的链接地址，下载这个图片
def DownImgP(domain,subDirList,subsubImgDir,bParse=False):
    for dir in subDirList:
        print("-------%s------" %dir)
        imgList=[]
        common.readLinkFile(dir+"\\img.txt",imgList)

        if not os.path.exists(dir + "\\%s\\"%subsubImgDir):
            os.makedirs(dir + "\\%s\\"%subsubImgDir)
        for img in imgList:
            if bParse:
                img=GetRealImg(img,".png")
            name=img.split("/")[-1]
            print(name)
            common.downImg(domain+img,dir+"\\%s\\%s" %(subsubImgDir,name))

#多线程方式下载
def MultiDownImgP(domain,subDirList,subsubImgDir,bParse=False):

    mth=MyThread.CMultiDownThread(10)

    for dir in subDirList:
        print("-------%s------" %dir)
        imgList=[]
        common.readLinkFile(dir+"\\img.txt",imgList)

        if not os.path.exists(dir + "\\%s\\"%subsubImgDir):
            os.makedirs(dir + "\\%s\\"%subsubImgDir)
        for img in imgList:
            if bParse:
                img=GetRealImg(img,".png")
            name=img.split("/")[-1]
            print(name)
            th=MyThread.CDownThread(domain+img,dir+"\\%s\\%s" %(subsubImgDir,name))
            while not mth.AddThread(th):
                sleep(5)
    mth.join()

#多线程方式下载
def MultiDownImg(domain,imgLinkList,dir):

    mth=MyThread.CMultiDownThread(10)

    for img in imgLinkList:
        imglink=domain+img["src"]
        imgName = imglink.split("/")[-1]

        imgPath=dir + "\\"+imgName
        print(imgPath)

        th = MyThread.CDownThread(imglink, imgPath)
        while not mth.AddThread(th):
            sleep(5)

    mth.join()


class Solution:
    def kidsWithCandies(self, candies: List[int], extraCandies: int) -> List[bool]:
        max_canday_value = max(candies)
        output = []
        for index in range(len(candies)):
            if(candies[index]+extraCandies >= max_canday_value):
                output.append(True)
            else:
                output.append(False)
        return output


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import os

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

# In[2]:

recs = ['ABD1', 'KIX']
names = ['Pdr1AD', 'Gal4', 'Ppr1', 'Pip2']

# In[6]:

for rec in recs:
    for name in names:
        cluster_path = '../' + rec + '/' + name + '/analysis/top500'
        all_path = '../' + rec + '/' + name + '/analysis/all'
        cluster_results = pd.read_csv(cluster_path, delimiter=' ')
        all_results = pd.read_csv(all_path, delimiter=' ')
        rmsd_thresh = max(all_results.sort_values('reweighted_sc').iloc[1:10].mean()['rms'], 2.0)
        print(f'Suggested clustering rmsd threshold for {rec:5} {name:8}: {rmsd_thresh:5.1f}')

# In[ ]:


import json

from django.contrib.auth.decorators import permission_required
from django.db import transaction, IntegrityError
from django.http import HttpResponse, HttpResponseBadRequest
from django.shortcuts import render, get_object_or_404
from django.utils.translation import ugettext_lazy as _
from django.views.generic import View

from bangoo.content.models import Content
from bangoo.decorators import class_view_decorator
from bangoo.navigation.menu.forms import MenuOrderForm, MenuRenameForm, MenuCreateForm
from bangoo.navigation.models import Menu

from .signals import menu_changed
from .utils import create_path

@permission_required('menu.add_menu')
def menu(request, template_name='navigation/menu/menu.html'):
    return render(request, template_name, {'nodes': Menu.objects.all(), 'form': MenuCreateForm()})

@class_view_decorator(permission_required('menu.add_menu'))
class ReorderMenuView(View):
    def post(self, request, *args, **kwargs):
        form = MenuOrderForm(request.POST)
        if form.is_valid():
            try:
                with transaction.atomic():
                    changed_paths = []

                    method = form.cleaned_data['method']
                    target_menu = form.cleaned_data['target']
                    source_menu = form.cleaned_data['source']

                    if method == 'move':
                        source_menu_parent = source_menu.parent
                        source_menu.move_to(target_menu, position='left')
                        source_menu.save()

                        menu_changed.send(sender=self.__class__,
                                          menu=source_menu,
                                          old_parent=source_menu_parent,
                                          new_parent=target_menu.parent)

                    if method == 'insert':
                        source_menu_parent = source_menu.parent
                        source_menu.move_to(target_menu)
                        source_menu.save()

                        menu_changed.send(sender=self.__class__,
                                          menu=source_menu,
                                          old_parent=source_menu_parent,
                                          new_parent=target_menu)

                    if method in {'move', 'insert'}:
                        for descendant in source_menu.get_descendants(include_self=True):
                            old_path = descendant.path

                            descendant.path = create_path(descendant)
                            descendant.save()

                            if descendant.plugin == 'bangoo.content':
                                content = Content.objects.language(descendant.language_code).filter(url=old_path).first()
                                content.url = descendant.path
                                content.save()

                            changed_paths.append({
                                'menu_id': descendant.id,
                                'path': descendant.path
                            })
                        return HttpResponse(json.dumps(changed_paths), content_type='application/json')
            except IntegrityError:
                return HttpResponseBadRequest(_("Failed to move the menu item because there is already one "
                                                "with the same name under the destination path."))
        else:
            reason = {
                'status': 'error',
                'reasons': dict([(k, form.error_class.as_text(v)) for k, v in list(form.errors.items())])
            }
            return HttpResponse(json.dumps(reason), content_type='application/json')

@class_view_decorator(permission_required('menu.add_menu'))
class RenameMenuView(View):
    def post(self, request, *args, **kwargs):
        menu_id = kwargs.pop('menu_id', None)
        menu = get_object_or_404(Menu, pk=menu_id)
        form = MenuRenameForm(request.POST, menu=menu)

        if form.is_valid():
            with transaction.atomic():
                changed_paths = []

                menu.title = form.cleaned_data['title']
                menu.path = create_path(menu)
                menu.save()

                changed_paths.append({
                    'menu_id': menu.id,
                    'path': menu.path
                })

                for descendant in menu.get_descendants():
                    descendant.path = create_path(descendant)
                    descendant.save()

                    changed_paths.append({
                        'menu_id': descendant.id,
                        'path': descendant.path
                    })

                return HttpResponse(json.dumps(changed_paths), content_type='application/json')
        else:
            reason = {
                'status': 'error',
                'menu_id': menu_id,
                'reasons': dict([(k, form.error_class.as_text(v)) for k, v in list(form.errors.items())])
            }
            return HttpResponse(json.dumps(reason), content_type='application/json')


# Q.1)

def validIPAddress(IP):

        def isIPv4(IP):
            ip = IP.split('.')
            for s in ip:
                if len(s) == 0 or len(s) > 3:
                    return 'Neither'
                if ((s[0] == '0' and len(s) != 1) or not s.isdigit() or int(s) > 255):
                    return 'Neither'
            return 'IPv4'

        def isIPv6(IP: str) -> str:
            ip = IP.split(':')
            for s in ip:
                if len(s) == 0 or len(s) > 4:
                    return "Neither"
                for c in s:
                     if c not in '0123456789abcdef':
                        return "Neither"
            return 'IPv6'

        if len(IP.split('.'))-1 == 3:
            return isIPv4(IP)
        elif len(IP.split(':'))-1 == 7:
            return isIPv6(IP.lower())
        return "Neither"

inp = input()
print(validIPAddress(inp))

# Q.2)

def letterCombinations(digits):
    if len(digits) == 0:
        return []
    characters = {2:"abc",3:"def",4:"ghi",5:"jkl",6:"mno",7:"pqrs",8:"tuv",9:"wxyz"}
    result = []
    solve(digits,characters,result)
    return result
def solve(digits, characters, result, current_string="",current_level = 0):
    if current_level == len(digits):
        result.append(current_string)
        return
    for i in characters[int(digits[current_level])]:
        solve(digits,characters,result,current_string+i,current_level+1)
inp = input()
print(letterCombinations(inp))

# Q.3)

def uniquePathsIII(grid):
    startx = starty = empty = 0
    for row in range(0, len(grid)):
        for col in range(0, len(grid[0])):
            cell = grid[row][col]
            if cell >= 0:
                empty += 1
            if cell == 1:
                startx, starty = row, col
    return dfs(grid, startx, starty, empty)

def dfs(grid, x, y, step):
    if grid[x][y] == 2:
        return step == 1
    ret = 0
    cur = grid[x][y]
    grid[x][y] = -2
    for dirx, diry in [(0, 1), (1, 0), (0, -1), (-1, 0)]:
        tx, ty = x + dirx, y + diry
        if 0 <= tx < len(grid) and 0 <= ty < len(grid[0]) and grid[tx][ty] >= 0:
            ret += dfs(grid, tx, ty, step - 1)
    grid[x][y] = cur
    return ret

lis = list(map(int,input().split()))
print(uniquePathsIII(lis))