import sys
_module = sys.modules[__name__]
del sys
config = _module
convert_weights = _module
demo = _module
eval = _module
lib = _module
build = _module
nms = _module
_ext = _module
nms = _module
build = _module
pth_nms = _module
nms_wrapper = _module
pycocotools = _module
coco = _module
cocoeval = _module
mask = _module
roi_align = _module
crop_and_resize = _module
build = _module
crop_and_resize = _module
roi_align = _module
network = _module
mask_rcnn = _module
postprocess = _module
ap = _module
visualize = _module
visualize_cap = _module
InputProcess = _module
preprocess = _module
coco_data_pipeline = _module
data_center = _module
test_data_loader = _module
realtime_demo = _module
tasks = _module
AnchorProcess = _module
BboxProcess = _module
bbox = _module
generate_anchors = _module
MaskProcess = _module
merge_task = _module
test_load = _module

from _paritybench_helpers import _mock_config, patch_functional
from unittest.mock import mock_open, MagicMock
from torch.autograd import Function
from torch.nn import Module
import abc, collections, copy, enum, functools, inspect, itertools, logging, math, matplotlib, numbers, numpy, pandas, queue, random, re, scipy, sklearn, string, tensorflow, time, torch, torchaudio, torchtext, torchvision, types, typing, uuid, warnings
import numpy as np
from torch import Tensor
patch_functional()
open = mock_open()
yaml = logging = sys = argparse = MagicMock()
ArgumentParser = argparse.ArgumentParser
_global_config = args = argv = cfg = config = params = _mock_config()
argparse.ArgumentParser.return_value.parse_args.return_value = _global_config
yaml.load.return_value = _global_config
sys.argv = _global_config
__version__ = '1.0.0'
xrange = range
wraps = functools.wraps

import itertools

import logging

import math

import random

import re

import time

from collections import OrderedDict

import numpy as np

import scipy.misc

import torch

import torch.nn as nn

import torch.nn.functional as F

from torch.autograd import Variable

from torch.nn.parameter import Parameter

import matplotlib

import matplotlib.pyplot as plt

import torch.backends.cudnn as cudnn

import torch.optim as optim

from torch.autograd import Function

from torch import nn

import torch.utils.model_zoo as model_zoo

import torch.utils.data as data

class CropAndResizeFunction(Function):

    def __init__(self, crop_height, crop_width, extrapolation_value=0):
        self.crop_height = crop_height
        self.crop_width = crop_width
        self.extrapolation_value = extrapolation_value

    def forward(self, image, boxes, box_ind):
        crops = torch.zeros_like(image)
        if image.is_cuda:
            _backend.crop_and_resize_gpu_forward(image, boxes, box_ind, self.extrapolation_value, self.crop_height, self.crop_width, crops)
        else:
            _backend.crop_and_resize_forward(image, boxes, box_ind, self.extrapolation_value, self.crop_height, self.crop_width, crops)
        self.im_size = image.size()
        self.save_for_backward(boxes, box_ind)
        return crops

    def backward(self, grad_outputs):
        boxes, box_ind = self.saved_tensors
        grad_outputs = grad_outputs.contiguous()
        grad_image = torch.zeros_like(grad_outputs).resize_(*self.im_size)
        if grad_outputs.is_cuda:
            _backend.crop_and_resize_gpu_backward(grad_outputs, boxes, box_ind, grad_image)
        else:
            _backend.crop_and_resize_backward(grad_outputs, boxes, box_ind, grad_image)
        return grad_image, None, None

class CropAndResize(nn.Module):
    """
    Crop and resize ported from tensorflow
    See more details on https://www.tensorflow.org/api_docs/python/tf/image/crop_and_resize
    """

    def __init__(self, crop_height, crop_width, extrapolation_value=0):
        super(CropAndResize, self).__init__()
        self.crop_height = crop_height
        self.crop_width = crop_width
        self.extrapolation_value = extrapolation_value

    def forward(self, image, boxes, box_ind):
        return CropAndResizeFunction(self.crop_height, self.crop_width, self.extrapolation_value)(image, boxes, box_ind)

class RoIAlign(nn.Module):

    def __init__(self, crop_height, crop_width, extrapolation_value=0):
        super(RoIAlign, self).__init__()
        self.crop_height = crop_height
        self.crop_width = crop_width
        self.extrapolation_value = extrapolation_value

    def forward(self, featuremap, boxes, box_ind):
        """
        RoIAlign based on crop_and_resize.
        See more details on https://github.com/ppwwyyxx/tensorpack/blob/6d5ba6a970710eaaa14b89d24aace179eb8ee1af/examples/FasterRCNN/model.py#L301
        :param featuremap: NxCxHxW
        :param boxes: Mx4 float box with (x1, y1, x2, y2) **without normalization**
        :param box_ind: M
        :return: MxCxoHxoW
        """
        x1, y1, x2, y2 = torch.split(boxes, 1, dim=1)
        spacing_w = (x2 - x1) / float(self.crop_width)
        spacing_h = (y2 - y1) / float(self.crop_height)
        image_height, image_width = featuremap.size()[2:4]
        nx0 = (x1 + spacing_w / 2 - 0.5) / float(image_width - 1)
        ny0 = (y1 + spacing_h / 2 - 0.5) / float(image_height - 1)
        nw = spacing_w * float(self.crop_width - 1) / float(image_width - 1)
        nh = spacing_w * float(self.crop_height - 1) / float(image_height - 1)
        boxes = torch.cat((ny0, nx0, ny0 + nh, nx0 + nw), 1)
        return CropAndResizeFunction(self.crop_height, self.crop_width, self.extrapolation_value)(featuremap, boxes, box_ind)

class Bottleneck(nn.Module):
    expansion = 4

    def __init__(self, inplanes, planes, stride=1, downsample=None):
        super(Bottleneck, self).__init__()
        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, stride=stride, bias=True)
        self.bn1 = nn.BatchNorm2d(planes, eps=0.001)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=True)
        self.bn2 = nn.BatchNorm2d(planes, eps=0.001)
        self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=True)
        self.bn3 = nn.BatchNorm2d(planes * 4, eps=0.001)
        self.relu = nn.ReLU(inplace=True)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        residual = x
        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)
        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)
        out = self.conv3(out)
        out = self.bn3(out)
        if self.downsample is not None:
            residual = self.downsample(x)
        out += residual
        out = self.relu(out)
        return out

class resnet_graph(nn.Module):

    def __init__(self, block, layers, stage5=False):
        self.inplanes = 64
        super(resnet_graph, self).__init__()
        self.stage5 = stage5
        self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=True)
        self.bn1 = nn.BatchNorm2d(64, eps=0.001)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=0, ceil_mode=True)
        self.layer1 = self._make_layer(block, 64, layers[0])
        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
        if self.stage5:
            self.layer4 = self._make_layer(block, 512, layers[3], stride=2)

    def _make_layer(self, block, planes, blocks, stride=1):
        downsample = None
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(nn.Conv2d(self.inplanes, planes * block.expansion, kernel_size=1, stride=stride, bias=True), nn.BatchNorm2d(planes * block.expansion, eps=0.001))
        layers = []
        layers.append(block(self.inplanes, planes, stride, downsample))
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(block(self.inplanes, planes))
        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)
        C1 = self.maxpool(x)
        C2 = self.layer1(C1)
        C3 = self.layer2(C2)
        C4 = self.layer3(C3)
        if self.stage5:
            C5 = self.layer4(C4)
        else:
            C5 = None
        return C1, C2, C3, C4, C5

class rpn_graph(nn.Module):

    def __init__(self, input_dims, anchors_per_location, anchor_stride):
        super(rpn_graph, self).__init__()
        self.rpn_conv_shared = nn.Conv2d(input_dims, 512, kernel_size=3, stride=anchor_stride, padding=1)
        self.rpn_class_raw = nn.Conv2d(512, 2 * anchors_per_location, kernel_size=1)
        self.rpn_bbox_pred = nn.Conv2d(512, 4 * anchors_per_location, kernel_size=1)

    def forward(self, x):
        shared = F.relu(self.rpn_conv_shared(x), True)
        x = self.rpn_class_raw(shared)
        rpn_class_logits = x.permute(0, 2, 3, 1).contiguous().view(x.size(0), -1, 2)
        rpn_probs = F.softmax(rpn_class_logits, dim=-1)
        x = self.rpn_bbox_pred(shared)
        rpn_bbox = x.permute(0, 2, 3, 1).contiguous().view(x.size(0), -1, 4)
        return rpn_class_logits, rpn_probs, rpn_bbox

def log2_graph(x):
    """Implementatin of Log2. pytorch doesn't have a native implemenation."""
    return torch.div(torch.log(x), math.log(2.0))

def ROIAlign(feature_maps, rois, config, pool_size, mode='bilinear'):
    """Implements ROI Align on the features.

    Params:
    - pool_shape: [height, width] of the output pooled regions. Usually [7, 7]
    - image_shape: [height, width, chanells]. Shape of input image in pixels

    Inputs:
    - boxes: [batch, num_boxes, (x1, y1, x2, y2)] in normalized
             coordinates. Possibly padded with zeros if not enough
             boxes to fill the array.
    - Feature maps: List of feature maps from different levels of the pyramid.
                    Each is [batch, channels, height, width]

    Output:
    Pooled regions in the shape: [batch, num_boxes, height, width, channels].
    The width and height are those specific in the pool_shape in the layer
    constructor.
    """
    """
    [  x2-x1             x1 + x2 - W + 1  ]
    [  -----      0      ---------------  ]
    [  W - 1                  W - 1       ]
    [                                     ]
    [           y2-y1    y1 + y2 - H + 1  ]
    [    0      -----    ---------------  ]
    [           H - 1         H - 1      ]
    """
    rois = rois.detach()
    crop_resize = CropAndResize(pool_size, pool_size, 0)
    roi_number = rois.size()[1]
    pooled = rois.data.new(config.IMAGES_PER_GPU * rois.size(1), 256, pool_size, pool_size).zero_()
    rois = rois.view(config.IMAGES_PER_GPU * rois.size(1), 4)
    x_1 = rois[:, (0)]
    y_1 = rois[:, (1)]
    x_2 = rois[:, (2)]
    y_2 = rois[:, (3)]
    roi_level = log2_graph(torch.div(torch.sqrt((y_2 - y_1) * (x_2 - x_1)), 224.0))
    roi_level = torch.clamp(torch.clamp(torch.add(torch.round(roi_level), 4), min=2), max=5)
    for i, level in enumerate(range(2, 6)):
        scaling_ratio = 2 ** level
        height = float(config.IMAGE_MAX_DIM) / scaling_ratio
        width = float(config.IMAGE_MAX_DIM) / scaling_ratio
        ixx = torch.eq(roi_level, level)
        box_indices = ixx.view(-1).int() * 0
        ix = torch.unsqueeze(ixx, 1)
        level_boxes = torch.masked_select(rois, ix)
        if level_boxes.size()[0] == 0:
            continue
        level_boxes = level_boxes.view(-1, 4)
        crops = crop_resize(feature_maps[i], torch.div(level_boxes, float(config.IMAGE_MAX_DIM))[:, ([1, 0, 3, 2])], box_indices)
        indices_pooled = ixx.nonzero()[:, (0)]
        pooled[(indices_pooled.data), :, :, :] = crops.data
    pooled = pooled.view(config.IMAGES_PER_GPU, roi_number, 256, pool_size, pool_size)
    pooled = Variable(pooled)
    return pooled

class fpn_classifier_graph(nn.Module):

    def __init__(self, num_classes, config):
        super(fpn_classifier_graph, self).__init__()
        self.num_classes = num_classes
        self.config = config
        self.mrcnn_class_conv1 = nn.Conv2d(256, 1024, kernel_size=self.config.POOL_SIZE, stride=1, padding=0)
        self.mrcnn_class_bn1 = nn.BatchNorm2d(1024, eps=0.001)
        self.mrcnn_class_conv2 = nn.Conv2d(1024, 1024, kernel_size=1, stride=1, padding=0)
        self.mrcnn_class_bn2 = nn.BatchNorm2d(1024, eps=0.001)
        self.mrcnn_class_logits = nn.Linear(1024, self.num_classes)
        self.mrcnn_bbox_fc = nn.Linear(1024, self.num_classes * 4)

    def forward(self, x, rpn_rois):
        start = time.time()
        x = ROIAlign(x, rpn_rois, self.config, self.config.POOL_SIZE)
        spend = time.time() - start
        None
        roi_number = x.size()[1]
        x = x.view(self.config.IMAGES_PER_GPU * roi_number, 256, self.config.POOL_SIZE, self.config.POOL_SIZE)
        x = self.mrcnn_class_conv1(x)
        x = self.mrcnn_class_bn1(x)
        x = F.relu(x, inplace=True)
        x = self.mrcnn_class_conv2(x)
        x = self.mrcnn_class_bn2(x)
        x = F.relu(x, inplace=True)
        shared = x.permute(0, 2, 3, 1).contiguous().view(x.size(0), -1)
        mrcnn_class_logits = self.mrcnn_class_logits(shared)
        mrcnn_probs = F.softmax(mrcnn_class_logits, dim=-1)
        x = self.mrcnn_bbox_fc(shared)
        mrcnn_bbox = x.view(x.size()[0], self.num_classes, 4)
        mrcnn_class_logits = mrcnn_class_logits.view(self.config.IMAGES_PER_GPU, roi_number, mrcnn_class_logits.size()[-1])
        mrcnn_probs = mrcnn_probs.view(self.config.IMAGES_PER_GPU, roi_number, mrcnn_probs.size()[-1])
        mrcnn_bbox = mrcnn_bbox.view(self.config.IMAGES_PER_GPU, roi_number, self.config.NUM_CLASSES, 4)
        return mrcnn_class_logits, mrcnn_probs, mrcnn_bbox

class build_fpn_mask_graph(nn.Module):

    def __init__(self, num_classes, config):
        """Builds the computation graph of the mask head of Feature Pyramid Network.

        rois: [batch, num_rois, (y1, x1, y2, x2)] Proposal boxes in normalized
              coordinates.
        feature_maps: List of feature maps from diffent layers of the pyramid,
                      [P2, P3, P4, P5]. Each has a different resolution.
        image_shape: [height, width, depth]
        pool_size: The width of the square feature map generated from ROI Pooling.
        num_classes: number of classes, which determines the depth of the results

        Returns: Masks [batch, roi_count, height, width, num_classes]
        """
        super(build_fpn_mask_graph, self).__init__()
        self.num_classes = num_classes
        self.config = config
        self.mrcnn_mask_conv1 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1)
        self.mrcnn_mask_bn1 = nn.BatchNorm2d(256, eps=0.001)
        self.mrcnn_mask_conv2 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1)
        self.mrcnn_mask_bn2 = nn.BatchNorm2d(256, eps=0.001)
        self.mrcnn_mask_conv3 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1)
        self.mrcnn_mask_bn3 = nn.BatchNorm2d(256, eps=0.001)
        self.mrcnn_mask_conv4 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1)
        self.mrcnn_mask_bn4 = nn.BatchNorm2d(256, eps=0.001)
        self.mrcnn_mask_deconv = nn.ConvTranspose2d(256, 256, kernel_size=2, stride=2)
        self.mrcnn_mask = nn.Conv2d(256, self.num_classes, kernel_size=1, stride=1)

    def forward(self, x, rpn_rois):
        x = ROIAlign(x, rpn_rois, self.config, self.config.MASK_POOL_SIZE)
        roi_number = x.size()[1]
        x = x.view(self.config.IMAGES_PER_GPU * roi_number, 256, self.config.MASK_POOL_SIZE, self.config.MASK_POOL_SIZE)
        x = self.mrcnn_mask_conv1(x)
        x = self.mrcnn_mask_bn1(x)
        x = F.relu(x, inplace=True)
        x = self.mrcnn_mask_conv2(x)
        x = self.mrcnn_mask_bn2(x)
        x = F.relu(x, inplace=True)
        x = self.mrcnn_mask_conv3(x)
        x = self.mrcnn_mask_bn3(x)
        x = F.relu(x, inplace=True)
        x = self.mrcnn_mask_conv4(x)
        x = self.mrcnn_mask_bn4(x)
        x = F.relu(x, inplace=True)
        x = self.mrcnn_mask_deconv(x)
        x = F.relu(x, inplace=True)
        x = self.mrcnn_mask(x)
        x = x.view(self.config.IMAGES_PER_GPU, roi_number, self.config.NUM_CLASSES, self.config.MASK_POOL_SIZE * 2, self.config.MASK_POOL_SIZE * 2)
        return x

def apply_box_deltas_graph(boxes, deltas):
    """Applies the given deltas to the given boxes.
    boxes: [N, 4] where each row is y1, x1, y2, x2
    deltas: [N, 4] where each row is [dy, dx, log(dh), log(dw)]
    """
    height = boxes[:, :, (2)] - boxes[:, :, (0)]
    width = boxes[:, :, (3)] - boxes[:, :, (1)]
    center_y = boxes[:, :, (0)] + 0.5 * height
    center_x = boxes[:, :, (1)] + 0.5 * width
    center_y += deltas[:, :, (0)] * height
    center_x += deltas[:, :, (1)] * width
    height *= torch.exp(deltas[:, :, (2)])
    width *= torch.exp(deltas[:, :, (3)])
    y1 = center_y - 0.5 * height
    x1 = center_x - 0.5 * width
    y2 = y1 + height
    x2 = x1 + width
    result = [y1, x1, y2, x2]
    return result

def clip_boxes_graph(boxes, window):
    """
    boxes: [N, 4] each row is y1, x1, y2, x2
    window: [4] in the form y1, x1, y2, x2
    """
    wy1, wx1, wy2, wx2 = window
    y1, x1, y2, x2 = boxes
    y1 = torch.max(torch.min(y1, wy2), wy1)
    x1 = torch.max(torch.min(x1, wx2), wx1)
    y2 = torch.max(torch.min(y2, wy2), wy1)
    x2 = torch.max(torch.min(x2, wx2), wx1)
    clipped = torch.stack([x1, y1, x2, y2], dim=2)
    return clipped

def generate_anchors(scales, ratios, shape, feature_stride, anchor_stride):
    """
    scales: 1D array of anchor sizes in pixels. Example: [32, 64, 128]
    ratios: 1D array of anchor ratios of width/height. Example: [0.5, 1, 2]
    shape: [height, width] spatial shape of the feature map over which
            to generate anchors.
    feature_stride: Stride of the feature map relative to the image in pixels.
    anchor_stride: Stride of anchors on the feature map. For example, if the
        value is 2 then generate anchors for every other feature map pixel.
    """
    scales, ratios = np.meshgrid(np.array(scales), np.array(ratios))
    scales = scales.flatten()
    ratios = ratios.flatten()
    heights = scales / np.sqrt(ratios)
    widths = scales * np.sqrt(ratios)
    shifts_y = np.arange(0, shape[0], anchor_stride) * feature_stride
    shifts_x = np.arange(0, shape[1], anchor_stride) * feature_stride
    shifts_x, shifts_y = np.meshgrid(shifts_x, shifts_y)
    box_widths, box_centers_x = np.meshgrid(widths, shifts_x)
    box_heights, box_centers_y = np.meshgrid(heights, shifts_y)
    box_centers = np.stack([box_centers_y, box_centers_x], axis=2).reshape([-1, 2])
    box_sizes = np.stack([box_heights, box_widths], axis=2).reshape([-1, 2])
    boxes = np.concatenate([box_centers - 0.5 * box_sizes, box_centers + 0.5 * box_sizes], axis=1)
    return boxes

def generate_pyramid_anchors(scales, ratios, feature_shapes, feature_strides, anchor_stride):
    """Generate anchors at different levels of a feature pyramid. Each scale
    is associated with a level of the pyramid, but each ratio is used in
    all levels of the pyramid.

    Returns:
    anchors: [N, (y1, x1, y2, x2)]. All generated anchors in one array. Sorted
        with the same order of the given scales. So, anchors of scale[0] come
        first, then anchors of scale[1], and so on.
    """
    anchors = []
    for i in range(len(scales)):
        anchors.append(generate_anchors(scales[i], ratios, feature_shapes[i], feature_strides[i], anchor_stride))
    return np.concatenate(anchors, axis=0)

def mrcnn_bbox_loss(target_bbox, target_class_ids, pred_bbox):
    """Loss for Mask R-CNN bounding box refinement.

    target_bbox: [batch, num_rois, (dy, dx, log(dh), log(dw))]
    target_class_ids: [batch, num_rois]. Integer class IDs.
    pred_bbox: [batch, num_rois, num_classes, (dy, dx, log(dh), log(dw))]
    """
    target_class_ids = target_class_ids.contiguous().view(-1)
    target_bbox = target_bbox.contiguous().view(-1, 4)
    pred_bbox = pred_bbox.contiguous().view(-1, pred_bbox.size()[2], 4)
    positive_roi_ix = torch.gt(target_class_ids, 0)
    positive_roi_class_ids = torch.masked_select(target_class_ids, positive_roi_ix)
    indices = target_class_ids
    loss = F.smooth_l1_loss(pred_bbox, target_bbox, size_average=True)
    return loss

def mrcnn_class_loss(target_class_ids, pred_class_logits, active_class_ids, config):
    """Loss for the classifier head of Mask RCNN.
    target_class_ids: [batch, num_rois]. Integer class IDs. Uses zero
        padding to fill in the array.
    pred_class_logits: [batch, num_rois, num_classes]
    active_class_ids: [batch, num_classes]. Has a value of 1 for
        classes that are in the dataset of the image, and 0
        for classes that are not in the dataset.
    """
    pred_class_logits = pred_class_logits.contiguous().view(-1, config.NUM_CLASSES)
    target_class_ids = target_class_ids.contiguous().view(-1).type(torch.LongTensor)
    loss = F.cross_entropy(pred_class_logits, target_class_ids, weight=None, size_average=True)
    return loss

def mrcnn_mask_loss(target_masks, target_class_ids, pred_masks_logits):
    """Mask binary cross-entropy loss for the masks head.

    target_masks: [batch, num_rois, height, width].
        A float32 tensor of values 0 or 1. Uses zero padding to fill array.
    target_class_ids: [batch, num_rois]. Integer class IDs. Zero padded.
    pred_masks: [batch, proposals, height, width, num_classes] float32 tensor
                with values from 0 to 1.
    """
    target_class_ids = target_class_ids.view(-1)
    loss = F.binary_cross_entropy_with_logits(pred_masks_logits, target_masks)
    return loss

def pth_nms(dets, thresh):
    """
  dets has to be a tensor
  """
    if not dets.is_cuda:
        x1 = dets[:, (0)]
        y1 = dets[:, (1)]
        x2 = dets[:, (2)]
        y2 = dets[:, (3)]
        scores = dets[:, (4)]
        areas = (x2 - x1 + 1) * (y2 - y1 + 1)
        order = scores.sort(0, descending=True)[1]
        keep = torch.LongTensor(dets.size(0))
        num_out = torch.LongTensor(1)
        nms.cpu_nms(keep, num_out, dets, order, areas, thresh)
        return keep[:num_out[0]]
    else:
        x1 = dets[:, (0)]
        y1 = dets[:, (1)]
        x2 = dets[:, (2)]
        y2 = dets[:, (3)]
        scores = dets[:, (4)]
        areas = (x2 - x1 + 1) * (y2 - y1 + 1)
        order = scores.sort(0, descending=True)[1]
        dets = dets[order].contiguous()
        keep = torch.LongTensor(dets.size(0))
        num_out = torch.LongTensor(1)
        nms.gpu_nms(keep, num_out, dets, thresh)
        return order[keep[:num_out[0]]].contiguous()

def nms(dets, thresh):
    """Dispatch to either CPU or GPU NMS implementations.
  Accept dets as tensor"""
    return pth_nms(dets, thresh)

def rpn_bbox_loss(target_bbox, rpn_match, rpn_bbox, config):
    """Return the RPN bounding box loss graph.

    config: the model config object.
    target_bbox: [batch, max positive anchors, (dy, dx, log(dh), log(dw))].
        Uses 0 padding to fill in unsed bbox deltas.
    rpn_match: [batch, anchors, 1]. Anchor match type. 1=positive,
               -1=negative, 0=neutral anchor.
    rpn_bbox: [batch, anchors, (dy, dx, log(dh), log(dw))]
    """
    indices = torch.eq(rpn_match, 1)
    rpn_bbox = torch.masked_select(rpn_bbox, indices)
    batch_counts = torch.sum(indices.float(), dim=1)
    outputs = []
    for i in range(config.IMAGES_PER_GPU):
        outputs.append(target_bbox[torch.LongTensor([i]), torch.arange(int(batch_counts[i].cpu().data.numpy()[0])).type(torch.LongTensor)])
    target_bbox = torch.cat(outputs, dim=0)
    loss = F.smooth_l1_loss(rpn_bbox, target_bbox, size_average=True)
    return loss

def rpn_class_loss(rpn_match, rpn_class_logits):
    """RPN anchor classifier loss.

    rpn_match: [batch, anchors, 1]. Anchor match type. 1=positive,
               -1=negative, 0=neutral anchor.
    rpn_class_logits: [batch, anchors, 2]. RPN classifier logits for FG/BG.
    """
    anchor_class = torch.eq(rpn_match, 1)
    indices = torch.ne(rpn_match, 0.0)
    rpn_class_logits = torch.masked_select(rpn_class_logits, indices)
    anchor_class = torch.masked_select(anchor_class, indices)
    rpn_class_logits = rpn_class_logits.contiguous().view(-1, 2)
    anchor_class = anchor_class.contiguous().view(-1).type(torch.LongTensor)
    loss = F.cross_entropy(rpn_class_logits, anchor_class, weight=None)
    return loss

def box_refinement(box, gt_box):
    """Compute refinement needed to transform box to gt_box.
    box and gt_box are [N, (y1, x1, y2, x2)]. (y2, x2) is
    assumed to be outside the box.
    """
    box = box.astype(np.float32)
    gt_box = gt_box.astype(np.float32)
    height = box[:, (2)] - box[:, (0)]
    width = box[:, (3)] - box[:, (1)]
    center_y = box[:, (0)] + 0.5 * height
    center_x = box[:, (1)] + 0.5 * width
    gt_height = gt_box[:, (2)] - gt_box[:, (0)]
    gt_width = gt_box[:, (3)] - gt_box[:, (1)]
    gt_center_y = gt_box[:, (0)] + 0.5 * gt_height
    gt_center_x = gt_box[:, (1)] + 0.5 * gt_width
    dy = (gt_center_y - center_y) / height
    dx = (gt_center_x - center_x) / width
    dh = np.log(gt_height / height)
    dw = np.log(gt_width / width)
    return np.stack([dy, dx, dh, dw], axis=1)

def compute_iou(box, boxes, box_area, boxes_area):
    """Calculates IoU of the given box with the array of the given boxes.
    box: 1D vector [y1, x1, y2, x2]
    boxes: [boxes_count, (y1, x1, y2, x2)]
    box_area: float. the area of 'box'
    boxes_area: array of length boxes_count.

    Note: the areas are passed in rather than calculated here for
          efficency. Calculate once in the caller to avoid duplicate work.
    """
    y1 = np.maximum(box[0], boxes[:, (0)])
    y2 = np.minimum(box[2], boxes[:, (2)])
    x1 = np.maximum(box[1], boxes[:, (1)])
    x2 = np.minimum(box[3], boxes[:, (3)])
    intersection = np.maximum(x2 - x1, 0) * np.maximum(y2 - y1, 0)
    union = box_area + boxes_area[:] - intersection[:]
    iou = intersection / union
    return iou

def build_detection_targets(rpn_rois, gt_class_ids, gt_boxes, gt_masks, config):
    """Generate targets for training Stage 2 classifier and mask heads.
    This is not used in normal training. It's useful for debugging or to train
    the Mask RCNN heads without using the RPN head.
    Inputs:
    rpn_rois: [N, (y1, x1, y2, x2)] proposal boxes.
    gt_class_ids: [instance count] Integer class IDs
    gt_boxes: [instance count, (y1, x1, y2, x2)]
    gt_masks: [height, width, instance count] Grund truth masks. Can be full
              size or mini-masks.
    Returns:
    rois: [TRAIN_ROIS_PER_IMAGE, (y1, x1, y2, x2)]
    class_ids: [TRAIN_ROIS_PER_IMAGE]. Integer class IDs.
    bboxes: [TRAIN_ROIS_PER_IMAGE, NUM_CLASSES, (y, x, log(h), log(w))]. Class-specific
            bbox refinments.
    masks: [TRAIN_ROIS_PER_IMAGE, height, width, NUM_CLASSES). Class specific masks cropped
           to bbox boundaries and resized to neural network output size.
    """
    assert rpn_rois.shape[0] > 0
    assert gt_class_ids.dtype == np.int32, 'Expected int but got {}'.format(gt_class_ids.dtype)
    assert gt_boxes.dtype == np.int32, 'Expected int but got {}'.format(gt_boxes.dtype)
    assert gt_masks.dtype == np.bool_, 'Expected bool but got {}'.format(gt_masks.dtype)
    instance_ids = np.where(gt_class_ids > 0)[0]
    assert instance_ids.shape[0] > 0, 'Image must contain instances.'
    gt_class_ids = gt_class_ids[instance_ids]
    gt_boxes = gt_boxes[instance_ids]
    gt_masks = gt_masks[:, :, (instance_ids)]
    rpn_roi_area = (rpn_rois[:, (2)] - rpn_rois[:, (0)]) * (rpn_rois[:, (3)] - rpn_rois[:, (1)])
    gt_box_area = (gt_boxes[:, (2)] - gt_boxes[:, (0)]) * (gt_boxes[:, (3)] - gt_boxes[:, (1)])
    overlaps = np.zeros((rpn_rois.shape[0], gt_boxes.shape[0]))
    for i in range(overlaps.shape[1]):
        gt = gt_boxes[i]
        overlaps[:, (i)] = compute_iou(gt, rpn_rois, gt_box_area[i], rpn_roi_area)
    rpn_roi_iou_argmax = np.argmax(overlaps, axis=1)
    rpn_roi_iou_max = overlaps[np.arange(overlaps.shape[0]), rpn_roi_iou_argmax]
    rpn_roi_gt_boxes = gt_boxes[rpn_roi_iou_argmax]
    rpn_roi_gt_class_ids = gt_class_ids[rpn_roi_iou_argmax]
    fg_ids = np.where(rpn_roi_iou_max > 0.5)[0]
    bg_ids = np.where(rpn_roi_iou_max < 0.5)[0]
    fg_roi_count = int(config.TRAIN_ROIS_PER_IMAGE * config.ROI_POSITIVE_RATIO)
    if fg_ids.shape[0] > fg_roi_count:
        keep_fg_ids = np.random.choice(fg_ids, fg_roi_count, replace=False)
    else:
        keep_fg_ids = fg_ids
    remaining = config.TRAIN_ROIS_PER_IMAGE - keep_fg_ids.shape[0]
    if bg_ids.shape[0] > remaining:
        keep_bg_ids = np.random.choice(bg_ids, remaining, replace=False)
    else:
        keep_bg_ids = bg_ids
    keep = np.concatenate([keep_fg_ids, keep_bg_ids])
    remaining = config.TRAIN_ROIS_PER_IMAGE - keep.shape[0]
    if remaining > 0:
        if keep.shape[0] == 0:
            bg_ids = np.where(rpn_roi_iou_max < 0.5)[0]
            assert bg_ids.shape[0] >= remaining
            keep_bg_ids = np.random.choice(bg_ids, remaining, replace=False)
            assert keep_bg_ids.shape[0] == remaining
            keep = np.concatenate([keep, keep_bg_ids])
        else:
            keep_extra_ids = np.random.choice(keep_bg_ids, remaining, replace=True)
            keep = np.concatenate([keep, keep_extra_ids])
    assert keep.shape[0] == config.TRAIN_ROIS_PER_IMAGE, "keep doesn't match ROI batch size {}, {}".format(keep.shape[0], config.TRAIN_ROIS_PER_IMAGE)
    rpn_roi_gt_boxes[(keep_bg_ids), :] = 0
    rpn_roi_gt_class_ids[keep_bg_ids] = 0
    rois = rpn_rois[keep]
    roi_gt_boxes = rpn_roi_gt_boxes[keep]
    roi_gt_class_ids = rpn_roi_gt_class_ids[keep]
    roi_gt_assignment = rpn_roi_iou_argmax[keep]
    bboxes = np.zeros((config.TRAIN_ROIS_PER_IMAGE, config.NUM_CLASSES, 4), dtype=np.float32)
    pos_ids = np.where(roi_gt_class_ids > 0)[0]
    bboxes[pos_ids, roi_gt_class_ids[pos_ids]] = box_refinement(rois[pos_ids], roi_gt_boxes[(pos_ids), :4])
    bboxes /= config.BBOX_STD_DEV
    masks = np.zeros((config.TRAIN_ROIS_PER_IMAGE, config.MASK_SHAPE[0], config.MASK_SHAPE[1], config.NUM_CLASSES), dtype=np.float32)
    for i in pos_ids:
        class_id = roi_gt_class_ids[i]
        assert class_id > 0, 'class id must be greater than 0'
        gt_id = roi_gt_assignment[i]
        class_mask = gt_masks[:, :, (gt_id)]
        if config.USE_MINI_MASK:
            placeholder = np.zeros(config.IMAGE_SHAPE[:2], dtype=bool)
            gt_y1, gt_x1, gt_y2, gt_x2 = gt_boxes[gt_id]
            gt_w = gt_x2 - gt_x1
            gt_h = gt_y2 - gt_y1
            placeholder[gt_y1:gt_y2, gt_x1:gt_x2] = np.round(scipy.misc.imresize(class_mask.astype(float), (gt_h, gt_w), interp='nearest') / 255.0).astype(bool)
            class_mask = placeholder
        y1, x1, y2, x2 = rois[i].astype(np.int32)
        m = class_mask[y1:y2, x1:x2]
        mask = scipy.misc.imresize(m.astype(float), config.MASK_SHAPE, interp='nearest') / 255.0
        masks[(i), :, :, (class_id)] = mask
    return rois, roi_gt_class_ids, bboxes, masks

def stage2_target(rpn_rois, gt_class_ids, gt_boxes, gt_masks, config):
    batch_rois = []
    batch_mrcnn_class_ids = []
    batch_mrcnn_bbox = []
    batch_mrcnn_mask = []
    for i in range(config.IMAGES_PER_GPU):
        rois, mrcnn_class_ids, mrcnn_bbox, mrcnn_mask = build_detection_targets(rpn_rois[i], gt_class_ids[i], gt_boxes[i], gt_masks[i], config)
        batch_rois.append(rois)
        batch_mrcnn_class_ids.append(mrcnn_class_ids)
        batch_mrcnn_bbox.append(mrcnn_bbox)
        batch_mrcnn_mask.append(mrcnn_mask)
    batch_rois = np.array(batch_rois)
    batch_mrcnn_class_ids = np.array(batch_mrcnn_class_ids)
    batch_mrcnn_bbox = np.array(batch_mrcnn_bbox)
    batch_mrcnn_mask = np.array(batch_mrcnn_mask)
    return batch_rois, batch_mrcnn_class_ids, batch_mrcnn_bbox, batch_mrcnn_mask

def to_variable(numpy_data, volatile=False):
    numpy_data = numpy_data.astype(np.float32)
    torch_data = torch.from_numpy(numpy_data).float()
    variable = Variable(torch_data, volatile=volatile)
    return variable

class MaskRCNN(nn.Module):
    """
    Encapsulates the Mask RCNN model functionality.

    """

    def __init__(self, config, mode='inference'):
        super(MaskRCNN, self).__init__()
        """
        config: A Sub-class of the Config class
        model_dir: Directory to save training logs and trained weights
        """
        self.config = config
        self.mode = mode
        self.resnet_graph = resnet_graph(Bottleneck, [3, 4, 23, 3], stage5=True)
        self.fpn_c5p5 = nn.Conv2d(512 * 4, 256, kernel_size=1, stride=1, padding=0)
        self.fpn_c4p4 = nn.Conv2d(256 * 4, 256, kernel_size=1, stride=1, padding=0)
        self.fpn_c3p3 = nn.Conv2d(128 * 4, 256, kernel_size=1, stride=1, padding=0)
        self.fpn_c2p2 = nn.Conv2d(64 * 4, 256, kernel_size=1, stride=1, padding=0)
        self.fpn_p2 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1)
        self.fpn_p3 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1)
        self.fpn_p4 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1)
        self.fpn_p5 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1)
        self.scale_ratios = [4, 8, 16, 32]
        self.fpn_p6 = nn.MaxPool2d(kernel_size=1, stride=2, padding=0, ceil_mode=False)
        self.anchors = generate_pyramid_anchors(self.config.RPN_ANCHOR_SCALES, self.config.RPN_ANCHOR_RATIOS, self.config.BACKBONE_SHAPES, self.config.BACKBONE_STRIDES, self.config.RPN_ANCHOR_STRIDE)
        self.anchors = self.anchors.astype(np.float32)
        self.rpn = rpn_graph(256, len(self.config.RPN_ANCHOR_RATIOS), self.config.RPN_ANCHOR_STRIDE)
        self.rpn_mask = build_fpn_mask_graph(config.NUM_CLASSES, config)
        self.rpn_class = fpn_classifier_graph(config.NUM_CLASSES, config)
        self.proposal_count = self.config.POST_NMS_ROIS_TRAINING if self.mode == 'training' else self.config.POST_NMS_ROIS_INFERENCE
        self._initialize_weights()

    def forward(self, x):
        start = time.time()
        saved_for_loss = []
        C1, C2, C3, C4, C5 = self.resnet_graph(x)
        resnet_time = time.time()
        None
        P5 = self.fpn_c5p5(C5)
        P4 = self.fpn_c4p4(C4) + F.upsample(P5, scale_factor=2, mode='bilinear')
        P3 = self.fpn_c3p3(C3) + F.upsample(P4, scale_factor=2, mode='bilinear')
        P2 = self.fpn_c2p2(C2) + F.upsample(P3, scale_factor=2, mode='bilinear')
        P2 = self.fpn_p2(P2)
        P3 = self.fpn_p3(P3)
        P4 = self.fpn_p4(P4)
        P5 = self.fpn_p5(P5)
        P6 = self.fpn_p6(P5)
        rpn_feature_maps = [P2, P3, P4, P5, P6]
        self.mrcnn_feature_maps = [P2, P3, P4, P5]
        rpn_class_logits_outputs = []
        rpn_class_outputs = []
        rpn_bbox_outputs = []
        for p in rpn_feature_maps:
            rpn_class_logits, rpn_probs, rpn_bbox = self.rpn(p)
            rpn_class_logits_outputs.append(rpn_class_logits)
            rpn_class_outputs.append(rpn_probs)
            rpn_bbox_outputs.append(rpn_bbox)
        rpn_class_logits = torch.cat(rpn_class_logits_outputs, dim=1)
        rpn_class = torch.cat(rpn_class_outputs, dim=1)
        rpn_bbox = torch.cat(rpn_bbox_outputs, dim=1)
        rpn_rois = self.proposal_layer(rpn_class, rpn_bbox)
        spend = time.time() - resnet_time
        None
        mrcnn_class_logits, mrcnn_class, mrcnn_bbox = self.rpn_class(self.mrcnn_feature_maps, rpn_rois)
        mrcnn_masks_logits = self.rpn_mask(self.mrcnn_feature_maps, rpn_rois)
        if self.mode == 'training':
            return [rpn_class_logits, rpn_class, rpn_bbox, rpn_rois, mrcnn_class_logits, mrcnn_class, mrcnn_bbox, mrcnn_masks_logits], [rpn_class_logits, rpn_class, rpn_bbox, rpn_rois, mrcnn_class_logits, mrcnn_class, mrcnn_bbox, mrcnn_masks_logits]
        else:
            return [rpn_class_logits, rpn_class, rpn_bbox, rpn_rois, mrcnn_class_logits, mrcnn_class, mrcnn_bbox, mrcnn_masks_logits]

    def proposal_layer(self, rpn_class, rpn_bbox):
        scores = rpn_class[:, :, (1)]
        deltas_mul = Variable(torch.from_numpy(np.reshape(self.config.RPN_BBOX_STD_DEV, [1, 1, 4]).astype(np.float32)))
        deltas = rpn_bbox * deltas_mul
        pre_nms_limit = min(6000, self.anchors.shape[0])
        scores, ix = torch.topk(scores, pre_nms_limit, dim=-1, largest=True, sorted=True)
        ix = torch.unsqueeze(ix, 2)
        ix = torch.cat([ix, ix, ix, ix], dim=2)
        deltas = torch.gather(deltas, 1, ix)
        _anchors = []
        for i in range(self.config.IMAGES_PER_GPU):
            anchors = Variable(torch.from_numpy(self.anchors.astype(np.float32)))
            _anchors.append(anchors)
        anchors = torch.stack(_anchors, 0)
        pre_nms_anchors = torch.gather(anchors, 1, ix)
        refined_anchors = apply_box_deltas_graph(pre_nms_anchors, deltas)
        height, width = self.config.IMAGE_SHAPE[:2]
        window = np.array([0, 0, height, width]).astype(np.float32)
        window = Variable(torch.from_numpy(window))
        refined_anchors_clipped = clip_boxes_graph(refined_anchors, window)
        refined_proposals = []
        for i in range(self.config.IMAGES_PER_GPU):
            indices = nms(torch.cat([refined_anchors_clipped.data[i], scores.data[i]], 1), 0.7)
            indices = indices[:self.proposal_count]
            indices = torch.stack([indices, indices, indices, indices], dim=1)
            indices = Variable(indices)
            proposals = torch.gather(refined_anchors_clipped[i], 0, indices)
            padding = self.proposal_count - proposals.size()[0]
            proposals = torch.cat([proposals, Variable(torch.zeros([padding, 4]))], 0)
            refined_proposals.append(proposals)
        rpn_rois = torch.stack(refined_proposals, 0)
        return rpn_rois

    @staticmethod
    def build_loss(saved_for_loss, ground_truths, config):
        saved_for_log = OrderedDict()
        predict_rpn_class_logits, predict_rpn_class, predict_rpn_bbox, predict_rpn_rois, predict_mrcnn_class_logits, predict_mrcnn_class, predict_mrcnn_bbox, predict_mrcnn_masks_logits = saved_for_loss
        batch_rpn_match, batch_rpn_bbox, batch_gt_class_ids, batch_gt_boxes, batch_gt_masks, active_class_ids = ground_truths
        rpn_rois = predict_rpn_rois.cpu().data.numpy()
        rpn_rois = rpn_rois[:, :, ([1, 0, 3, 2])]
        batch_rois, batch_mrcnn_class_ids, batch_mrcnn_bbox, batch_mrcnn_mask = stage2_target(rpn_rois, batch_gt_class_ids, batch_gt_boxes, batch_gt_masks, config)
        batch_mrcnn_mask = batch_mrcnn_mask.transpose(0, 1, 4, 2, 3)
        batch_mrcnn_class_ids = to_variable(batch_mrcnn_class_ids)
        batch_mrcnn_bbox = to_variable(batch_mrcnn_bbox)
        batch_mrcnn_mask = to_variable(batch_mrcnn_mask)
        rpn_cls_loss = rpn_class_loss(batch_rpn_match, predict_rpn_class_logits)
        rpn_reg_loss = rpn_bbox_loss(batch_rpn_bbox, batch_rpn_match, predict_rpn_bbox, config)
        stage2_reg_loss = mrcnn_bbox_loss(batch_mrcnn_bbox, batch_mrcnn_class_ids, predict_mrcnn_bbox)
        stage2_cls_loss = mrcnn_class_loss(batch_mrcnn_class_ids, predict_mrcnn_class_logits, active_class_ids, config)
        stage2_mask_loss = mrcnn_mask_loss(batch_mrcnn_mask, batch_mrcnn_class_ids, predict_mrcnn_masks_logits)
        total_loss = rpn_cls_loss + rpn_reg_loss + stage2_cls_loss + stage2_reg_loss + stage2_mask_loss
        saved_for_log['rpn_cls_loss'] = rpn_cls_loss.data[0]
        saved_for_log['rpn_reg_loss'] = rpn_reg_loss.data[0]
        saved_for_log['stage2_cls_loss'] = stage2_cls_loss.data[0]
        saved_for_log['stage2_reg_loss'] = stage2_reg_loss.data[0]
        saved_for_log['stage2_mask_loss'] = stage2_mask_loss.data[0]
        saved_for_log['total_loss'] = total_loss.data[0]
        return total_loss, saved_for_log

    def _initialize_weights(self):
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2.0 / n))
                if m.bias is not None:
                    m.bias.data.zero_()
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
            elif isinstance(m, nn.Linear):
                m.weight.data.normal_(0, 0.01)
                m.bias.data.zero_()

import torch
from torch.nn import MSELoss, ReLU
from _paritybench_helpers import _mock_config, _mock_layer, _paritybench_base, _fails_compile

TESTCASES = [
    # (nn.Module, init_args, forward_args, jit_compiles)
    (rpn_graph,
     lambda: ([], {'input_dims': 4, 'anchors_per_location': 4, 'anchor_stride': 1}),
     lambda: ([torch.rand([4, 4, 4, 4])], {}),
     True),
]

class Test_soeaver_Pytorch_Mask_RCNN(_paritybench_base):
    def test_000(self):
        self._check(*TESTCASES[0])


import io
import csv
import pandas as pd
import numpy as np

from astropy.coordinates import SkyCoord
from astropy.table import Table
import astropy.units as u

from fink_science.tester import regular_unit_tests

MANGROVE_COLS = [
    'HyperLEDA_name',
    '2MASS_name',
    'lum_dist',
    'ang_dist'
]

def cross_match_astropy(pdf, catalog_ztf, catalog_other, radius_arcsec=None):
    """ Crossmatch two catalogs

    Parameters
    ----------
    pdf: pd.DataFrame
        Input alert dataframe
    catalog_ztf: SkyCoord
        Catalog of ZTF objects
    catalog_other: SkyCoord
        Catalog of other objects
    radius_arcsec: pd.Series or None
        If None, default is 1.5 arcsec
    """
    # cross-match
    idx, d2d, d3d = catalog_other.match_to_catalog_sky(catalog_ztf)

    # set separation length
    if radius_arcsec is None:
        radius_arcsec = 1.5
    else:
        radius_arcsec = float(radius_arcsec.values[0])

    sep_constraint = d2d.degree < radius_arcsec / 3600.0

    catalog_matches = np.unique(pdf['candid'].values[idx[sep_constraint]])

    # identify position of matches in the input dataframe
    pdf_matches = pd.DataFrame(
        {
            'candid': np.array(catalog_matches, dtype=np.int64),
            'match': True
        }
    )
    pdf_merge = pd.merge(pdf, pdf_matches, how='left', on='candid')

    mask = pdf_merge['match'].apply(lambda x: x is True)

    # Now get types for these
    catalog_ztf_merge = SkyCoord(
        ra=np.array(pdf_merge.loc[mask, 'ra'].values, dtype=float) * u.degree,
        dec=np.array(pdf_merge.loc[mask, 'dec'].values, dtype=float) * u.degree
    )

    # cross-match
    idx2, d2d2, d3d2 = catalog_ztf_merge.match_to_catalog_sky(catalog_other)

    return pdf_merge, mask, idx2

def extract_mangrove(filename):
    """ Read the Mangrove catalog and extract useful columns

    Parameters
    ----------
    filename: str
        Path to the Mangrove catalog (parquet file)

    Returns
    ----------
    out: pd.Series, pd.Series, pd.Series
        (ra, dec, Source Name )from the catalog

    Examples
    ----------
    >>> import os
    >>> curdir = os.path.dirname(os.path.abspath(__file__))
    >>> filename = curdir + '/../data/catalogs/mangrove_filtered.parquet'
    >>> ra, dec, payload = extract_mangrove(filename)
    """
    pdf = pd.read_parquet(filename)

    cols = ['HyperLEDA_name', '2MASS_name', 'lum_dist', 'ang_dist']

    for col_ in cols:
        # stringify
        pdf[col_] = pdf[col_].astype(str)

    payload = pdf[cols].to_dict(orient='records')

    return pdf['ra'], pdf['dec'], payload

def extract_4lac(filename_h, filename_l):
    """ Read the 4LAC DR3 catalogs and extract useful columns

    Parameters
    ----------
    filename{i}: str
        Path to the high and low latitudes catalogs (fits file)

    Returns
    ----------
    out: pd.Series, pd.Series, pd.Series
        (ra, dec, Source Name )from the catalog

    Examples
    ----------
    >>> import os
    >>> curdir = os.path.dirname(os.path.abspath(__file__))
    >>> catalog_h = curdir + '/../data/catalogs/table-4LAC-DR3-h.fits'
    >>> catalog_l = curdir + '/../data/catalogs/table-4LAC-DR3-l.fits'
    >>> ra, dec, sourcename = extract_4lac(catalog_h, catalog_l)
    """
    dat = Table.read(filename_h, format='fits')
    pdf_4lac_h = dat.to_pandas()

    dat = Table.read(filename_l, format='fits')
    pdf_4lac_l = dat.to_pandas()

    pdf_4lac = pd.concat([pdf_4lac_h, pdf_4lac_l])

    # decode as str
    pdf_4lac['Source_Name'] = pdf_4lac['Source_Name'].apply(lambda x: x.decode())

    return pdf_4lac['RAJ2000'], pdf_4lac['DEJ2000'], pdf_4lac['Source_Name']

def extract_3hsp(filename):
    """ Read the 3HSP catalog and extract useful columns

    Parameters
    ----------
    filename: str
        Path to the catalog (csv file)

    Returns
    ----------
    out: pd.Series, pd.Series, pd.Series
        (ra, dec, Source Name )from the catalog

    Examples
    ----------
    >>> import os
    >>> curdir = os.path.dirname(os.path.abspath(__file__))
    >>> catalog = curdir + '/../data/catalogs/3hsp.csv'
    >>> ra, dec, sourcename = extract_3hsp(catalog)
    """
    pdf_3hsp = pd.read_csv(filename, header=0)

    # remove white spaces around column names
    pdf_3hsp = pdf_3hsp.rename(columns={i: i.strip() for i in pdf_3hsp.columns})

    # convert RA/Dec into degrees
    pdf_3hsp['R.A.'] = pdf_3hsp['R.A.'].apply(lambda x: x.strip().replace('"', ''))
    pdf_3hsp['Dec'] = pdf_3hsp['Dec'].apply(lambda x: x.strip().replace('"', ''))

    coord = SkyCoord(
        pdf_3hsp[['R.A.', 'Dec']].apply(lambda x: '{} {}'.format(*x), axis=1).values,
        unit=(u.hourangle, u.deg)
    )

    pdf_3hsp['ra'] = coord.ra.deg
    pdf_3hsp['dec'] = coord.dec.deg

    # format correctly names!
    pdf_3hsp['3HSP Source name'] = pdf_3hsp['3HSP Source name'].apply(lambda x: x.strip().replace('"', ''))

    return pdf_3hsp['ra'], pdf_3hsp['dec'], pdf_3hsp['3HSP Source name']

def extract_gcvs(filename):
    """ Read the gcvs catalog and extract useful columns

    Parameters
    ----------
    filename: str
        Path to the catalog (parquet file)

    Returns
    ----------
    out: pd.Series, pd.Series, pd.Series
        ra, dec, VarType from the catalog

    Examples
    ----------
    >>> import os
    >>> curdir = os.path.dirname(os.path.abspath(__file__))
    >>> catalog = curdir + '/../data/catalogs/gcvs.parquet'
    >>> ra, dec, vartype = extract_gcvs(catalog)
    """
    pdf = pd.read_parquet(filename)
    return pdf['ra'], pdf['dec'], pdf['VarType']

def extract_vsx(filename):
    """ Read the vsx catalog and extract useful columns

    Parameters
    ----------
    filename: str
        Path to the catalog (parquet file)

    Returns
    ----------
    out: pd.Series, pd.Series, pd.Series
        ra, dec, VarType from the catalog

    Examples
    ----------
    >>> import os
    >>> curdir = os.path.dirname(os.path.abspath(__file__))
    >>> catalog = curdir + '/../data/catalogs/vsx.parquet'
    >>> ra, dec, vtype = extract_vsx(catalog)
    """
    pdf_vsx = pd.read_parquet(filename)
    return pdf_vsx['RAJ2000'], pdf_vsx['DEJ2000'], pdf_vsx['VType']

def generate_csv(s: str, lists: list) -> str:
    """ Make a string (CSV formatted) given lists of data and header.
    Parameters
    ----------
    s: str
        String which will contain the data.
        Should initially contain the CSV header.
    lists: list of lists
        List containing data.
        Length of `lists` must correspond to the header.

    Returns
    ----------
    s: str
        Updated string with one row per line.

    Examples
    ----------
    >>> header = "toto,tata\\n"
    >>> lists = [[1, 2], ["cat", "dog"]]
    >>> table = generate_csv(header, lists)
    >>> print(table)
    toto,tata
    1,"cat"
    2,"dog"
    <BLANKLINE>
    """
    output = io.StringIO()
    writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC)
    _ = [writer.writerow(row) for row in zip(*lists)]
    return s + output.getvalue().replace('\r', '')

if __name__ == "__main__":
    """ Execute the test suite """

    # Run the test suite
    regular_unit_tests(globals())


from bs4 import BeautifulSoup
import selenium
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import pandas as pd
driver = webdriver.Chrome(r"C:\Users\mrmap\PycharmProjects\Internship-Database\chromedriver.exe")
def loadBloombergJobs(searchTerm, driver): # extracts jobInfo from Bloomberg website through webdriver
    driver.get('https://careers.bloomberg.com/job/search?qf=' + searchTerm)
    """
    jobSearchBox = driver.find_element_by_id("search-text-input")
    jobSearchBox.send_keys(searchTerm)
    jobSearchBox.click()
    jobSearchBox.send_keys(Keys.ENTER)
    """
    driver.implicitly_wait(10)
    pageSource = driver.page_source
    soup = BeautifulSoup(pageSource, "html.parser")
    jobSoup = soup.find(class_="job-results-list")
    return jobSoup
def extractJobTitleBloomberg(jobElem): # extracts job title from web data
    titleElem = jobElem.find(class_="job-results-name")
    title = titleElem.text.strip()
    return title
def extractJobLocationBloomberg(jobElem): # extracts job location from web data
    locationElem = jobElem.find(class_="job-results-city")
    location = locationElem.text.strip()
    return location
def extractJobDateBloomberg(jobElem): # extracts job date from web date
    dateElem = jobElem.find(class_="job-results-date")
    date = dateElem.text.strip()
    return date
def extractJobLinkBloomberg(jobElem): # extracts job link from web data
    link = "careers.bloomberg.com" + jobElem.find('a')['href']
    return link
def saveJobsToExcel(jobsList, filename): # update based on what format we want
    jobs = pd.DataFrame(jobsList)
    jobs.to_excel(filename)
def findJobsFrom(website, searchTerm, desiredCharacs, filename="BloombergJobs.xlsx"): # loads Bloomberg website, extracts data and saves it
    if website == 'Bloomberg':
        jobSoup = loadBloombergJobs(searchTerm, driver)
        jobsList, numListings = extractJobInformationBloomberg(jobSoup, desiredCharacs)
        saveJobsToExcel(jobsList, filename)
        print('{} new job postings retrieved. Stored in {}.'.format(numListings, filename))
def extractJobInformationBloomberg(jobSoup, desiredCharacs): # extracts data by searching for desired information per job
    jobElems = jobSoup.find_all(class_="job-results-section")
    cols = [] # creates columns array
    extractedInfo = [] # creates array of info
    company = []
    cols.append('Company')
    for jobElem in jobElems:
        company.append("Bloomberg")
    extractedInfo.append(company)
    if 'titles' in desiredCharacs: # if titles in desired characs, creates title array
        titles = []
        cols.append('Titles')
        for jobElem in jobElems:
            titles.append(extractJobTitleBloomberg(jobElem))
        extractedInfo.append(titles)
    if 'locations' in desiredCharacs: # if locations in desired characs, creates locations array
        locations = []
        cols.append('Locations')
        for jobElem in jobElems:
            locations.append(extractJobLocationBloomberg(jobElem))
        extractedInfo.append(locations)
    if 'dates' in desiredCharacs: # if dates in desired characs, creates dates array
        dates = []
        cols.append('Dates')
        for jobElem in jobElems:
            dates.append(extractJobDateBloomberg(jobElem))
        extractedInfo.append(dates)
    if 'links' in desiredCharacs: # if links in desired characs, creates links array
        links = []
        cols.append('Links')
        for jobElem in jobElems:
            links.append(extractJobLinkBloomberg(jobElem))
        extractedInfo.append(links)
    jobsList = {}
    for j in range(len(cols)):
        jobsList[cols[j]] = extractedInfo[j]
    numListings = len(extractedInfo[0])
    return jobsList, numListings
desiredCharacs = ['titles', 'locations', 'dates', 'links']
findJobsFrom('Bloomberg', 'summer intern', desiredCharacs)


import pandas as pd
from os.path import join
import numpy as np
import torch.utils.data as data
import sys
from sklearn.neighbors import NearestNeighbors

from datasets.datahelpers import default_loader, imresize

default_cities = {
    'train': ["zurich", "london", "boston", "melbourne", "amsterdam","helsinki",
              "tokyo","toronto","saopaulo","moscow","trondheim","paris","bangkok",
              "budapest","austin","berlin","ottawa","phoenix","goa","amman","nairobi","manila"],
    'val': ["cph", "sf"],
    'test': ["miami","athens","buenosaires","stockholm","bengaluru","kampala"]
}

default_cities_debug = {
    'train': ["zurich", "london"],
    'val': ["cph", "sf"],
    'test': ["miami"]
}

class BaseDataset(data.Dataset):

    def __init__(self, name, mode='train', imsize=None, transform=None, loader=default_loader, posDistThr=10, negDistThr=25, root_dir = 'data', cities = ''):

        if cities == 'debug':
            self.cities = default_cities_debug[mode]
        elif cities in default_cities:
            self.cities = default_cities[cities]
        elif cities == '':
            self.cities = default_cities[mode]
        else:
            self.cities = cities.split(',')

        self.dbImages = []
        self.qImages = []
        self.qidxs = []
        self.pidxs = []
        self.clusters = []

        # hyper-parameters
        self.posDistThr = posDistThr
        self.negDistThr = negDistThr
        self.imsize = imsize

        self.transform = transform
        self.loader = loader

        # flags
        self.name = name
        self.exclude_panos = True
        self.mode = mode

        # other
        self.transform = transform

        # load data
        for city in self.cities:
            print("=====> {}".format(city))

            subdir = 'test' if city in default_cities['test'] else 'train_val'

            # get len of images from cities so far for indexing
            _lenQ = len(self.qImages)
            _lenDb = len(self.dbImages)

            # when GPS / UTM is available
            if self.mode in ['train', 'val', 'test']:

                # load query / database data
                qData = pd.read_csv(join(root_dir, subdir, city, 'query', 'postprocessed.csv'), index_col = 0)
                dbData = pd.read_csv(join(root_dir, subdir, city, 'database', 'postprocessed.csv'), index_col = 0)

                # filter based on panorama data
                if self.exclude_panos:

                    # load query / database data
                    qDataRaw = pd.read_csv(join(root_dir, subdir, city, 'query', 'raw.csv'), index_col = 0)
                    dbDataRaw = pd.read_csv(join(root_dir, subdir, city, 'database', 'raw.csv'), index_col = 0)

                    qData = qData.loc[(qDataRaw['pano'] == False).values, :]
                    dbData = dbData.loc[(dbDataRaw['pano'] == False).values, :]

                # append image keys with full path
                self.qImages.extend([join(root_dir, subdir, city, 'query', 'images', key + '.jpg') for key in qData['key'].values])
                self.dbImages.extend([join(root_dir, subdir, city, 'database', 'images', key + '.jpg') for key in dbData['key'].values])

                # utm coordinates
                utmQ = qData[['easting', 'northing']].values.reshape(-1,2)
                utmDb = dbData[['easting', 'northing']].values.reshape(-1,2)

                # find positive images for training
                neigh = NearestNeighbors(algorithm = 'brute')
                neigh.fit(utmDb)
                _, pI = neigh.radius_neighbors(utmQ, self.posDistThr)

                if mode == 'train':
                    _, nI = neigh.radius_neighbors(utmQ, self.negDistThr)

                for qidx in range(len(qData)):

                    # the query image has at least one positive
                    if len(pI[qidx]) > 0:

                        self.qidxs.append(qidx + _lenQ)
                        self.pidxs.append([p + _lenDb for p in pI[qidx]])

                        # in training we have two thresholds, one for finding positives and one for finding images that we are certain are negatives.
                        if self.mode == 'train':

                            self.clusters.append([n + _lenDb for n in nI[qidx]])

                self.utmQ = np.concatenate([self.utmQ, utmQ], axis=0) if hasattr(self, 'utmQ') else utmQ
                self.utmDb = np.concatenate([self.utmDb, utmDb], axis=0) if hasattr(self, 'utmDb') else utmDb

            # when GPS / UTM / pano info is not available
            elif self.mode in ['test2']:
                raise NotImplementedError

            # if a combination of cities, task and subtask is chosen, where there are no query/database images, then exit
            if len(self.dbImages) == 0:
                print("Exiting...")
                print("A combination of cities, task and subtask have been chosen, where there are no query/database images.")
                print("Try choosing a different subtask or more cities")
                sys.exit()

        # cast to np.arrays for indexing during training
        self.dbImages = np.asarray(self.dbImages)
        self.qImages = np.asarray(self.qImages)
        self.qidxs = np.asarray(self.qidxs)
        self.pidxs = np.asarray(self.pidxs, dtype=object)
        self.clusters = np.asarray(self.clusters, dtype=object)

    def __len__(self):
        raise NotImplementedError

    def __getitem__(self, index):

        raise NotImplementedError

    def __repr__(self):
        fmt_str = self.__class__.__name__ + '\n'
        fmt_str += '    Name and mode: {} {}\n'.format(self.name, self.mode)
        fmt_str += '    Number of images: {}\n'.format(len(self.dbImages))
        fmt_str += '    Number of training tuples: {}\n'.format(len(self.qpool))
        fmt_str += '    Number of negatives per tuple: {}\n'.format(self.nnum)
        fmt_str += '    Number of tuples processed in an epoch: {}\n'.format(self.qsize)
        fmt_str += '    Pool size for negative remining: {}\n'.format(self.poolsize)
        tmp = '    Transforms (if any): '
        fmt_str += '{0}{1}\n'.format(tmp, self.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp)))
        return fmt_str

class TrainDataset(BaseDataset):

    def __init__(self, name, mode, imsize=None, transform=None, loader=default_loader, posDistThr=10, negDistThr=25, root_dir = 'data', cities = ''):
        super().__init__(name, mode, imsize, transform, loader, posDistThr, negDistThr, root_dir, cities)

    def __len__(self):

        return len(self.qidxs)

    def __getitem__(self, index):

        qidx =  self.qidxs[index]
        pidx =  self.pidxs[index]

        pidx =  np.random.choice(pidx, 1)[0]

        qpath, utmQ = self.qImages[qidx], self.utmQ[qidx]
        ppath, utmDb = self.dbImages[pidx], self.utmDb[pidx]

        output = []
        output.append(self.loader(qpath))
        output.append(self.loader(ppath))

        target = []
        target.append(utmQ)
        target.append(utmDb)

        if self.imsize is not None:
            output = [imresize(img, self.imsize) for img in output]

        if self.transform is not None:
            output = [self.transform(output[i]) for i in range(len(output))]

        return output, target

class TestDataset(BaseDataset):

    def __init__(self, name, mode, imsize=None, transform=None, loader=default_loader, posDistThr=10, negDistThr=25, root_dir = 'data', cities = ''):
        super().__init__(name, mode, imsize, transform, loader, posDistThr, negDistThr, root_dir, cities)

    def __len__(self):
        # the dataset is the queries followed by the database images
        return len(self.qidxs) + len(self.dbImages)

    def __getitem__(self, index):

        if index < len(self.qidxs):
            path = self.qImages[self.qidxs[index]]
            utm = self.utmQ[self.qidxs[index]]
            index = [index, -1]
        else:
            path = self.dbImages[index - len(self.qidxs)]
            utm = self.utmDb[index - len(self.qidxs)]
            index = [-1, index - len(self.qidxs)]

        img = self.loader(path)

        if self.imsize is not None:
            img = imresize(img, self.imsize)

        if self.transform is not None:
            img = self.transform(img)

        return img, index, utm


#Step1: Creating connection

def Read ():
    Myfile = open("Myfile.txt",'r')

    #Reading Data using Same connection
    MyData = Myfile.readline()
    FinalData = Myfile.readlines()
    print(MyData)
    for line in FinalData:
        print(line)
    Myfile.close()

def Write():
    MyCon = open("Myfile.txt",'a')
    Msg = input("Enter your message")
    MyCon.writelines(Msg)
    MyCon.close()

def Create ():
    MyFile = open("Myprofile.txt",'w+')
    MyFile.write("Created file at run time ...!!!!")
    MyFile.close()

    MyFile1 = open("Myprofile.txt",'r')
    msg = MyFile1.readlines()
    print(msg)
    MyFile1.close()
"""Read()
Write()
Read()"""
Create()


def get_rules(data, with_quantifier):
    rules = {}
    for rule in data:
        rule_buffer = rule.split('contain')
        outer_bag = rule_buffer[0][0:len(rule_buffer[0])-6]
        inner_bags = (rule_buffer[1].strip()[0:len(rule_buffer[1])-2]).split(',')
        rules[outer_bag] = []
        if inner_bags[0] == 'no other bags':
            continue
        else:
            for bag in inner_bags:
                stripped_bag = bag.strip()
                quantifier = int(stripped_bag[0])
                color = stripped_bag[1:len(stripped_bag)].split('bag')[0].strip()
                if with_quantifier:
                    rules[outer_bag].append([quantifier, color])
                else:
                    rules[outer_bag].append(color)
    return rules

def get_outer_bags(rules, result, inner_bag):
    for rule in rules:
        if (inner_bag in rules[rule]) and (rule not in result):
            result.append(rule)
            result = get_outer_bags(rules, result, rule)
    return result

def get_total_number_of_bags(rules, outer_bag):
    result = 1
    for bag in rules[outer_bag]:
        result += bag[0] * get_total_number_of_bags(rules, bag[1])
    return result

def challenge1(file_path):
    with open(file_path, "r") as file:
        data = file.read().strip().split("\n")

    rules = get_rules(data, False)

    result = len(get_outer_bags(rules, [], 'shiny gold'))
    return result

def challenge2(file_path):
    with open(file_path, "r") as file:
        data = file.read().strip().split("\n")

    rules = get_rules(data, True)

    # the result includes the outer bag as well.
    # that is subtracted to get the total number of contained bags.
    result = get_total_number_of_bags(rules, 'shiny gold') - 1
    return result


"""
Utility methods for model serialization
"""

# Authors: Nikolay Melnik <nmelnik@upwork.com>

from datetime import datetime, date, time

class JsonSerializableMixin(object):  # pragma: no cover

    def __json__(self):
        """
        Converts all the properties of the object into a dict for use in json.
        You can define the following in your class

        _json_eager_load :
            list of which child classes need to be eagerly loaded. This applies
            to one-to-many relationships defined in SQLAlchemy classes.

        _base_blacklist :
            top level blacklist list of which properties not to include in JSON

        _json_blacklist :
            blacklist list of which properties not to include in JSON

        :return: dictionary ready to be jsonified
        :rtype: <dict>
        """
        props = {}

        # grab the json_eager_load set, if it exists
        # use set for easy 'in' lookups
        json_eager_load = set(getattr(self, '_json_eager_load', []))
        # now load the property if it exists
        # (does this issue too many SQL statements?)
        for prop in json_eager_load:
            getattr(self, prop, None)

        # we make a copy because the dict will change if the database
        # is updated / flushed
        options = self.__dict__.copy()

        # setup the blacklist
        # use set for easy 'in' lookups
        blacklist = set(getattr(self, '_base_blacklist', []))
        # extend the base blacklist with the json blacklist
        blacklist.update(getattr(self, '_json_blacklist', []))

        for key in options:
            # skip blacklisted, private and SQLAlchemy properties
            if key in blacklist or key.startswith(('__', '_sa_')):
                continue

            # format and date/datetime/time properties to isoformat
            obj = getattr(self, key)
            if isinstance(obj, (datetime, date, time)):
                props[key] = obj.isoformat()
            else:
                # get the class property value
                attr = getattr(self, key)
                # let see if we need to eagerly load it
                if key in json_eager_load:
                    # this is for SQLAlchemy foreign key fields that
                    # indicate with one-to-many relationships
                    if not hasattr(attr, 'pk') and attr:
                        # jsonify all child objects
                        attr = [x.__json__() for x in attr]
                else:
                    # convert all non integer strings to string or if
                    # string conversion is not possible, convert it to
                    # Unicode
                    if attr and not isinstance(attr, (int, float)):
                        try:
                            attr = str(attr)
                        except UnicodeEncodeError:
                            attr = unicode(attr)  # .encode('utf-8')

                props[key] = attr

        return props


from som.vector import Vector

_INITIAL_CAPACITY = 16

class Entry:
    def __init__(self, hash_, key, value, next_):
        self.hash = hash_
        self.key = key
        self.value = value
        self.next = next_

    def match(self, hash_, key):
        return self.hash == hash_ and key == self.key

def _hash(key):
    if key is None:
        return 0

    hash_ = key.custom_hash()
    return hash_ ^ hash_ >> 16

class Dictionary:
    def __init__(self, size=_INITIAL_CAPACITY):
        self._buckets = [None] * size
        self._size = 0

    def size(self):
        return self._size

    def is_empty(self):
        return self._size == 0

    def _get_bucket_idx(self, hash_):
        return (len(self._buckets) - 1) & hash_

    def _get_bucket(self, hash_):
        return self._buckets[self._get_bucket_idx(hash_)]

    def at(self, key):
        hash_ = _hash(key)
        e = self._get_bucket(hash_)

        while e is not None:
            if e.match(hash_, key):
                return e.value
            e = e.next
        return None

    def contains_key(self, key):
        hash_ = _hash(key)
        e = self._get_bucket(hash_)

        while e is not None:
            if e.match(hash_, key):
                return True
            e = e.next
        return False

    def at_put(self, key, value):
        hash_ = _hash(key)
        i = self._get_bucket_idx(hash_)

        current = self._buckets[i]

        if current is None:
            self._buckets[i] = self._new_entry(key, value, hash_)
            self._size += 1
        else:
            self._insert_bucket_entry(key, value, hash_, current)

        if self._size > len(self._buckets):
            self._resize()

    def _new_entry(self, key, value, hash_):
        return Entry(hash_, key, value, None)

    def _insert_bucket_entry(self, key, value, hash_, head):
        current = head

        while True:
            if current.match(hash_, key):
                current.value = value
                return

            if current.next is None:
                self._size += 1
                current.next = self._new_entry(key, value, hash_)
                return

            current = current.next

    def _resize(self):
        old_storage = self._buckets

        new_storage = [None] * len(old_storage) * 2
        self._buckets = new_storage
        self._transfer_entries(old_storage)

    def _transfer_entries(self, old_storage):
        for i in range(len(old_storage)):  # pylint: disable=consider-using-enumerate
            current = old_storage[i]
            if current is not None:
                old_storage[i] = None

                if current.next is None:
                    self._buckets[current.hash & (len(self._buckets) - 1)] = current
                else:
                    self._split_bucket(old_storage, i, current)

    def _split_bucket(self, old_storage, i, head):
        lo_head = None
        lo_tail = None
        hi_head = None
        hi_tail = None
        current = head

        while current is not None:
            if (current.hash & len(old_storage)) == 0:
                if lo_tail is None:
                    lo_head = current
                else:
                    lo_tail.next = current
                lo_tail = current
            else:
                if hi_tail is None:
                    hi_head = current
                else:
                    hi_tail.next = current
                hi_tail = current
            current = current.next

        if lo_tail is not None:
            lo_tail.next = None
            self._buckets[i] = lo_head
        if hi_tail is not None:
            hi_tail.next = None
            self._buckets[i + len(old_storage)] = hi_head

    def remove_all(self):
        self._buckets = [None] * len(self._buckets)
        self._size = 0

    def get_keys(self):
        keys = Vector(self._size)
        for i in range(len(self._buckets)):
            current = self._buckets[i]
            while current is not None:
                keys.append(current.key)
                current = current.next
        return keys

    def get_values(self):
        values = Vector(self._size)
        for i in range(len(self._buckets)):
            current = self._buckets[i]
            while current is not None:
                values.append(current.value)
                current = current.next
        return values


# from sklearn.cluster import FeatureAgglomeration
from Basic_trainer import *

from torchvision.utils import make_grid as make_grid
from torchvision.utils import save_image
import pandas as pd
from collections import defaultdict

class Sparse_tester(Trainer_Basic):
    '''Sparse_CT tester
    args:
        opt: the trainer opt
        cttool: cttool init by traner
        save_fig: save pic or not
    '''
    def __init__(self, opt, net,
                test_win = None,
                data_range = 1,
                **kwargs):
        super(Sparse_tester, self).__init__()
        self.opt = opt
        self.net = net
        self.sparse_angle = self.opt.sparse_angle
        self.cttool = CT_Preprocessing()
        if test_win is not None:
            assert isinstance(test_win, list)
        else:
            test_win = [(3000,500),(500,50),(800,-600)]
        self.test_win = test_win
        self.save_fig = self.opt.tester_save_image

        # 彩色绘图
        if 'rgb_mode' in kwargs:
            self.rgb_mode = kwargs['rgb_mode']
            if 'rgb_dict' in kwargs:
                self.rgb_dict = kwargs['rgb_dict']
        else:
            self.rgb_mode = False

        self.data_range = data_range

        # tester data
        self.matrix_dict = defaultdict(dict)
        self.save_dir = os.path.join(self.opt.tester_save_path, self.opt.tester_save_name)
        # 【TODO add tester_save_path and tester_save_name in args】
        if not os.path.exists(self.save_dir):
            os.mkdir(self.save_dir)
        self.saved_slice = 0
        self.seed_torch(seed=1)

    def seed_torch(self, seed=1):
        random.seed(seed)
        os.environ['PYTHONHASHSEED'] = str(seed)
        np.random.seed(seed)
        torch.manual_seed(seed)
        torch.cuda.manual_seed(seed)
        torch.cuda.manual_seed_all(seed)
        torch.backends.cudnn.benchmark = False
        torch.backends.cudnn.deterministic = True

    def prepare_dataset(self, ):
        print('here')
        opt = self.opt
        print(opt.dataset_name)
        if opt.dataset_name == 'deepleision':
            print('Evaluating on deeplesion Dataset')
            self.test_dataset = Deep_Lesion_Dataset(opt.dataset_path, 'test', dataset_shape = opt.dataset_shape)
        elif opt.dataset_name == 'AAPM':
            print('Evaluating on AAPM Myo Dataset')
            self.test_dataset = AAPM_Myo_Dataset(opt.dataset_path, 'test', dataset_shape = opt.dataset_shape)
        else:
            raise NotImplemented(f'dataset {opt.dataset_name} not implemented')
        self.test_loader = DataLoader(self.test_dataset, batch_size=1, num_workers=1,)

    # 【here】所有方法的forward
    def test_model(self, data):
        if 'Glo' in self.opt.network:
            sparse_ct_miu, dense_ct_miu, gt_ct_miu = self.net.sparse_pre_processing_w_multiview(data, )
            test_dict = {'sps': sparse_ct_miu}
            student_output_miu, student_latent = self.net([sparse_ct_miu, dense_ct_miu, 'student'])
            test_dict = {
                'out': student_output_miu,
            }
            return gt_ct_miu, test_dict
        else:
            raise NotImplementedError(f'network {self.opt.network} not implemented')

    def tester(self,):
        # a simple tester
        if self.opt.network != 'fbp':
            self.load_model(strict=True)
        self.net = self.net.cuda()
        self.prepare_dataset()
        self.net = self.net.eval()
        pbar = tqdm.tqdm(self.test_loader, ncols = 60)
        with torch.no_grad():
            for i, data in enumerate(pbar):
                data = data.cuda()
                gt_ct_miu, test_dict = self.test_model(data)
                self.batch_sparse_tester(gt_ct_miu, **test_dict)
        stat_matrix_dic = self.stat_matrix_dic(self.matrix_dict)
        stat_matrix_df = self.get_df_from_dict(stat_matrix_dic)
        stat_matrix_df.to_csv(os.path.join(self.save_dir, self.opt.network + str(self.opt.sparse_angle) +'_dic.csv'),)
        myprint(stat_matrix_df)

    def batch_sparse_tester(self, gt_ct=None, **kwargs):
        # gt_ct is the gt, use the **first element** in kwargs to calc matrix
        # if save_fig is True, save other pic to window_ct
        assert gt_ct is not None and len(kwargs)>0
        key_list = [k for k in kwargs.keys()]
        batch, _, _, _ = kwargs[key_list[0]].size()
        if batch == 1:
            self.sparse_tester(gt_ct, **kwargs)
        else:
            for b in range(batch):
                single_kwargs = {}
                for key in key_list:
                    single_kwargs[key] = kwargs[key][b:b+1]
                self.sparse_tester(gt_ct[b:b+1], single_kwargs)

    def sparse_tester(self, gt_ct=None, **kwargs):
        for (width, center) in self.test_win:
            calc_flag = -1
            for key, value in kwargs.items():
                if calc_flag < 0:
                    normhu_gtct = self.get_window_norm(gt_ct)
                    normhu_value = self.get_window_norm(value)
                    win_gtct = self.get_window_x_from_norm(normhu_gtct,width=width, center=center)
                    win_value = self.get_window_x_from_norm(normhu_value,width=width, center=center)
                    rmse, psnr, ssim = compute_measure(win_value, win_gtct, self.data_range)
                    try:
                        self.matrix_dict[f'{width}_{center}']['psnr'].append(psnr)
                        self.matrix_dict[f'{width}_{center}']['ssim'].append(ssim)
                        self.matrix_dict[f'{width}_{center}']['rmse'].append(rmse)
                    except:
                        self.matrix_dict[f'{width}_{center}']['psnr'] = []
                        self.matrix_dict[f'{width}_{center}']['ssim'] = []
                        self.matrix_dict[f'{width}_{center}']['rmse'] = []
                        self.matrix_dict[f'{width}_{center}']['psnr'].append(psnr)
                        self.matrix_dict[f'{width}_{center}']['ssim'].append(ssim)
                        self.matrix_dict[f'{width}_{center}']['rmse'].append(rmse)
                    calc_flag += 1
                # save image
                normhu_value = self.get_window_norm(value)
                win_value = self.get_window_x_from_norm(normhu_value,width=width, center=center)
                # if self.save_fig and (width == 2000 and center == 0):
                if self.save_fig:
                    psnr_str = '%.2f'%(psnr)
                    ssim_str = '%.2f'%(100*ssim)
                    self.save(win_value, f'{key}[{self.opt.sparse_angle}]_[{width},{center}]_p{psnr_str}_s{ssim_str}')

            if self.opt.network == 'fbp' and self.save_fig:
                normhu_gtct = self.get_window_norm(gt_ct)
                win_gtct = self.get_window_x_from_norm(normhu_gtct,width=width, center=center)
                self.save(win_gtct, f'{"gt"}_[{width},{center}]_p100_s100')
                normhu_spct = self.get_window_norm(value)
                win_spct = self.get_window_x_from_norm(normhu_spct,width=width, center=center)
                self.save(win_spct, f'{"sp"}_[{width},{center}]_p100_s100')
        self.saved_slice += 1

    def save(self, value, name):
        save_dir = self.save_dir
        len_num = int(np.log10(len(self.test_dataset))) + 1
        slice_str = str(self.saved_slice).rjust(len_num, '0')
        net_str = self.opt.network.lower()
        fullname = f'{slice_str}_{name}_{net_str}.png'
        save_path = os.path.join(save_dir, fullname)
        save_image(value, save_path, normalize=False)

    # analysis tool
    def get_mean_from_dict(self, ):
        pass

    def get_df_from_dict(self, double_dict):
        if double_dict is None:
            double_dict = self.matrix_dict
        index_key, columns_key = self.get_columns_index_key(double_dict)

        # init a dataframe
        df_matrix = pd.DataFrame(columns=columns_key, index = index_key)
        for i in columns_key:
            for j in index_key:
                df_matrix.loc[j,i] = double_dict[j][i]
        return df_matrix

    def get_columns_index_key(self, double_dic,):
        columns_key_t = double_dic.keys()
        columns_key = []
        for t in columns_key_t:
            columns_key.append(t) #.cpu().numpy().item()
        columns_key = self.num_str_sort(columns_key)
        index_key = double_dic[columns_key[0]].keys()
        return columns_key, index_key

    # ---- cttools ----
    def get_window_x_from_norm(self, x, width, center):
        HUtensor = self.cttool.back_window_transform(x,width=3000,center=500)
        window_tensor = self.cttool.window_transform(HUtensor,width=width,center=center)
        return window_tensor

    def get_window_norm(self, x):
        return self.cttool.window_transform(self.cttool.miu2HU(x))

    def get_window_normHU(self, x):
        return self.cttool.window_transform(x)

    @staticmethod
    def stat_matrix_dic(matrix_dic):
        stat_matrix_dict = defaultdict(dict)
        for columns_key in matrix_dic.keys():
            for row_key in matrix_dic[columns_key]:
                mean_value = np.mean(matrix_dic[columns_key][row_key])
                std_value = np.std(matrix_dic[columns_key][row_key])
                min_value = np.min(matrix_dic[columns_key][row_key])
                max_value = np.max(matrix_dic[columns_key][row_key])
                stat_matrix_dict[columns_key][f'avg_{row_key}'] = mean_value
                stat_matrix_dict[columns_key][f'std_{row_key}'] = std_value
                stat_matrix_dict[columns_key][f'min_{row_key}'] = min_value
                stat_matrix_dict[columns_key][f'max_{row_key}'] = max_value
        return stat_matrix_dict

    @staticmethod
    def num_str_sort(num_str_list):
        num_list = []
        str_list = []
        for i in num_str_list:
            if type(i) != type('str'):
                num_list.append(i)
            else:
                str_list.append(i)
        num_list.sort()
        str_list.sort()
        num_str_list = []
        num_str_list.extend(num_list)
        num_str_list.extend(str_list)
        return num_str_list

    # ----------- no use ----------
    def fit(self,):
        pass
    def train(self,):
        pass
    def val(self,):
        pass


from typing import Dict, Any
import json
import requests

def get_dependencies() -> Dict:
    """
    Export data from file as dictionary from json
    :return: dependencies in dict type
    """

    # JSON file
    f = open('./data/deps.json', "r")
    # Reading from file
    data = json.loads(f.read())
    # Closing file
    f.close()
    return data

def get_version_data(dependency_name: str) -> Dict:
    """
    This method handles an external api request to extract data for current dependency
    :param dependency_name: name of dependency
    :return: dependency's latest version
    """
    # request on external api pull data for dependency
    response = requests.get(f'https://registry.npmjs.org/{dependency_name}')
    # transform to json type
    res = response.json()
    # return the latest version of dependency
    return res["dist-tags"]["latest"]

def version_update_status(current_version: str, newest_version: str) -> Dict:
    """
    Process controls the need of updating semantic versions
    :param current_version: the installed version
    :param newest_version: the newest version has published
    :return: status boolean codes for each semantic version
    """
    response = {}
    versions = ["major", "minor", "patch"]

    # iterate through each version based semantic versioning
    for index, current_version in enumerate(current_version.split(".")):
        # check if any version needs update so update the response status
        if current_version < newest_version.split(".")[index]:
            response[versions[index]] = True
        else:
            response[versions[index]] = False

    return response


#kwadraty liczb od 0 do 20

liczby = []

for x in range(21):
    liczby.append(x**2)

print(liczby)

#uzywajac list comprehension:
#jedna linijka kodu

numerki = [x**2 for x in range(21)]

print(numerki)


# import pandas as pd

# data = pd.read_csv('data.csv')

# print(data)

# -------------------------------------

# formas de pegar apenas o que está na coluna 'job_title'

# 1 - lógica normal

data = [
  {'job_title': 'estagiario', 'min_salary': 1200, 'max_salary': 2300},
  {'job_title': 'junior', 'min_salary': 3000, 'max_salary': 5000},
  {'job_title': 'pleno', 'min_salary': 6000, 'max_salary': 8000},
  {'job_title': 'senior', 'min_salary': 9000, 'max_salary': 12000},
]

conjunto = set()
for value in data:
    conjunto.add(value['job_title'])

print(conjunto)

# -------------------------------------------------
# 2 - forma pythonica - com compreension

data = [
  {'job_title': 'estagiario', 'min_salary': 1200, 'max_salary': 2300},
  {'job_title': 'junior', 'min_salary': 3000, 'max_salary': 5000},
  {'job_title': 'pleno', 'min_salary': 6000, 'max_salary': 8000},
  {'job_title': 'senior', 'min_salary': 9000, 'max_salary': 12000},
]

compreension = [value['job_title'] for value in data]
print(compreension)

# -------------------------------------------------
# 3 - compreension para filtrar valores maiores que 6000

data = [
  {'job_title': 'estagiario', 'min_salary': 1200, 'max_salary': 2300},
  {'job_title': 'junior', 'min_salary': 3000, 'max_salary': 5000},
  {'job_title': 'pleno', 'min_salary': 6000, 'max_salary': 8000},
  {'job_title': 'senior', 'min_salary': 9000, 'max_salary': 12000},
]

salario_maior = [value['max_salary'] for value in data if value['max_salary'] >= 6000]
print(salario_maior)

# -------------------------------------------------
# 4 - usando HOF para filtrar salarios

data = [
  {'job_title': 'estagiario', 'min_salary': 1200, 'max_salary': 2300},
  {'job_title': 'junior', 'min_salary': 3000, 'max_salary': 5000},
  {'job_title': 'pleno', 'min_salary': 6000, 'max_salary': 8000},
  {'job_title': 'senior', 'min_salary': 9000, 'max_salary': 12000},
]

filter_salary = list(filter(lambda x : x['max_salary'] > 6000, data))

print(filter_salary)

# -------------------------------------------------
# 5 - filtro básico

teste = [1,2,3,4,5,6,7]
res = list(filter(lambda x : x > 5, teste))
print(res)

# ----------------------------------------------------
# 6 - map

teste = [1,2,3,4,5,6,7]
res = list(map(lambda x : x * 5, teste))
print(res)

# ------------------
# filtrando valores minimos e maximo

# import pandas as pd

# data = pd.read_csv('data.csv')
# result = data['max_salary']

# result.max()
# result.min()

Ciencia-Computação/Bloco 33 - Introdução a Python/revisao/revisao.py


from keras.applications import ResNet50
from keras.utils import np_utils, conv_utils
from keras.backend.common import normalize_data_format
from keras import layers,models
from keras.layers import Flatten, Dense, Input
from keras.preprocessing import image
from keras.applications.resnet50 import preprocess_input, decode_predictions
import numpy as np
import os
import cv2

def eachFile(filepath):                 #将目录内的文件名放入列表中
    pathDir =  os.listdir(filepath)
    out = []
    for allDir in pathDir:
        child = allDir.encode('gbk')    # .decode('gbk')是解决中文显示乱码问题
        child = child.decode('gbk')
        out.append(child)
    return out

#从文件夹中获取图像数据
def get_data(train_left=0.0, train_right=0.7, train_all=0.7, resize=True, data_format=None,
             t=''):  # 从文件夹中获取图像数据
    # file_name = os.path.join(pic_dir_out, data_name + t + '_' + str(train_left) + '_' + str(train_right) + '_' + str(
    #     Width) + "X" + str(Height) + ".h5")
    #
    # if os.path.exists(file_name):  # 判断之前是否有存到文件中
    #     f = h5py.File(file_name, 'r')
    #     if t == 'train':
    #         X_train = f['X_train'][:]
    #         y_train = f['y_train'][:]
    #         f.close()
    #         return (X_train, y_train)
    #     elif t == 'test':
    #         X_test = f['X_test'][:]
    #         y_test = f['y_test'][:]
    #         f.close()
    #         return (X_test, y_test)
    #     else:
    #         return
    data_format = normalize_data_format(data_format)
    pic_dir_set = eachFile(pic_dir)
    X_train = []
    y_train = []
    X_test = []
    y_test = []
    label = 0
    for pic_dir_name in pic_dir_set:

        if not os.path.isdir(os.path.join(pic_dir, pic_dir_name)):# 如果不是文件夹，跳过
            continue
        pic_set = eachFile(os.path.join(pic_dir, pic_dir_name))# 图片列表
        pic_index = 0
        train_count = int(len(pic_set) * train_all) # 训练样本数目
        train_l = int(len(pic_set) * train_left)
        train_r = int(len(pic_set) * train_right)
        for pic_name in pic_set:
            if not os.path.isfile(os.path.join(pic_dir, pic_dir_name, pic_name)):
                continue
            img = cv2.imread(os.path.join(pic_dir, pic_dir_name, pic_name))
            if img is None:
                continue
            if (resize):
                img = cv2.resize(img, (Width, Height))
                img = img.reshape(-1, Width, Height, 3)
            if (pic_index < train_count):# 统计数量
                if t == 'train':
                    if (pic_index >= train_l and pic_index < train_r):
                        X_train.append(img)
                        y_train.append(label)
            else:
                if t == 'test':
                    X_test.append(img)
                    y_test.append(label)
            pic_index += 1
        if not len(pic_set) == 0:
            label += 1

    # f = h5py.File(file_name, 'w')
    return (X_train, y_train)
    # if t == 'train':
    #     X_train = np.concatenate(X_train, axis=0)
    #     y_train = np.array(y_train)
    #     f.create_dataset('X_train', data=X_train)
    #     f.create_dataset('y_train', data=y_train)
    #     f.close()
    #     return (X_train, y_train)
    # elif t == 'test':
    #     X_test = np.concatenate(X_test, axis=0)
    #     y_test = np.array(y_test)
    #     f.create_dataset('X_test', data=X_test)
    #     f.create_dataset('y_test', data=y_test)
    #     f.close()
    #     return (X_test, y_test)
    # else:
    #     return

def main():
    global Width, Height, pic_dir
###### 超参数设置#####################################
    Width = 280 # 设置训练集图像宽高
    Height = 280
    num_classes = 5 # 类别设置
    pic_dir = r'E:\细胞\x100\5类_tdf/'  # 数据集的路径

######################################################
    (x_train, y_train) = get_data(0.0,0.7,data_format='channels_last',t='train')
    y_train = np_utils.to_categorical(y_train, num_classes)
    input_tensor = Input(shape=(280, 280, 3))

    model = ResNet50(include_top=False,
                     weights='imagenet',
                     input_shape=(280,280,3),
                     pooling='max',
                     )
    x = model.output
    x = layers.Dense(5, activation='softmax')(x)
    model = models.Model(inputs=model.input, outputs=x)
    for layer in model.layers:
        layer.trainable = False

    model.compile(optimizer='adam', loss='categorical_crossentropy')
    for i, layer in enumerate(model.layers):
       print(i, layer.name)
    # model.summary()
if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""
Created on Wed Mar 17 14:50:06 2021

@author: Michael J. Murphy
"""
# Calculate meq/L for Stiff diagrams

import pandas as pd

class MeqCalc:

    def __init__(self, data):
        self.data = data
        self.df1 = pd.read_excel(self.data, engine='openpyxl') # All data
        self.df2 = self.df1[["LocCode", "Field_ID", "Sampled_Date-Time", "ChemName", "Conc_num"]] # Required columns
        self.chems = sorted(self.df2.ChemName.unique().tolist()) # Expected analytes and sorting: ['Alkalinity (Bicarbonate)','Calcium','Chloride','Magnesium','Potassium','Sodium','Sulfate']

    def calc_meq(self, ions = ["HCO3-", "Ca++", "Cl-", "Mg++", "Na+ + K+", "Na+ + K+", "SO4--"], charges = [-1, 2, -1, 2, 1, 1, -2],
                 molar_masses = [61.0168, 40.078, 35.453, 24.305, 39.0983, 22.99, 96.06]):
        chems = self.chems
        chem_zip = zip(chems, charges, molar_masses, ions)
        # Write mass charge to dataframe for conversion to meq/L
        self.df2["mass_charge"] = 0
        self.df2["meq/L"] = 0
        self.df2["ion"] = ''
        for chem, charge, mass, ion in chem_zip:
            self.df2.loc[self.df2.ChemName==chem, 'mass_charge'] = mass / charge
            self.df2.loc[self.df2.ChemName==chem, "meq/L"] = self.df2.Conc_num / self.df2.mass_charge
            self.df2.loc[self.df2.ChemName==chem, "ion"] = ion
        self.df2["key"] = self.df2["LocCode"] + "_" + self.df2["ChemName"]
        return self.df2

    def compare_samples(self, data_pre):
        self.data_pre = data_pre
        _df_new = self.calc_meq() # New data to compare to previous
        _meq_pre = MeqCalc(self.data_pre)
        _meq_pre.calc_meq()
        _df_pre = _meq_pre.df2 # Previous data
        # Test to make sure data is called in correct order
        if not _df_pre["Sampled_Date-Time"].max() < _df_new["Sampled_Date-Time"].max():

            raise Exception("Data frames are not in correct order; call 'new' data first, old data in compare_samples()")

        df_joined = _df_new.merge(_df_pre, on="key", how ="inner", suffixes=('_new', '_old'))
        df_joined["percent_change"] = ((df_joined["meq/L_new"].abs() - df_joined["meq/L_old"].abs()) / df_joined["meq/L_old"].abs())*100
        return df_joined

    def report(self, data_pre, project="test"):
        _df_rpt = self.compare_samples(data_pre)
        _df_rpt = _df_rpt[["LocCode_new", "ChemName_new", "ion_new", "Sampled_Date-Time_new", "meq/L_new", "Sampled_Date-Time_old", "meq/L_old", "percent_change"]]
        _date_from = _df_rpt["Sampled_Date-Time_old"].min().strftime('%b_%y')
        _date_to = _df_rpt["Sampled_Date-Time_new"].max().strftime('%b_%y')
        _df_rpt.columns = ["Location", "Chemical Name", "Ion", "Sampled Date, Current Quarter", "Ionic Charge Concentration, Current Quarter (meq/L)", "Sampled Date, Previous Quarter",
                           "Ionic Charge Concentration, Previous Quarter (meq/L)", "Percent Change in Charge Concentration"]
        _writer = pd.ExcelWriter(project + "_ion_summary_" + _date_from + "_to_"+ _date_to + ".xls")
        _df_rpt.to_excel(_writer)
        _writer.save()

        return _df_rpt


"""
Admin site bindings for profiles
"""
from django.contrib import admin
from django.contrib.auth.admin import UserAdmin as ContribUserAdmin
from django.contrib.auth import get_user_model
from django.utils.translation import gettext_lazy as _
from hijack_admin.admin import HijackUserAdminMixin

from profiles.models import LegalAddress, Profile

User = get_user_model()

class UserLegalAddressInline(admin.StackedInline):
    """Admin view for the legal address"""

    model = LegalAddress
    classes = ["collapse"]

    fieldsets = (
        (
            None,
            {
                "fields": (
                    ("first_name", "last_name"),
                    "street_address_1",
                    "street_address_2",
                    "street_address_3",
                    "street_address_4",
                    "street_address_5",
                    ("city", "state_or_territory", "postal_code"),
                    "country",
                )
            },
        ),
    )

    def has_delete_permission(self, request, obj=None):
        return False

class UserProfileInline(admin.StackedInline):
    """Admin view for the profile"""

    model = Profile
    classes = ["collapse"]

    def has_delete_permission(self, request, obj=None):
        return True

class UserAdmin(ContribUserAdmin, HijackUserAdminMixin):
    """Admin views for user"""

    fieldsets = (
        (None, {"fields": ("username", "password", "last_login")}),
        (_("Personal Info"), {"fields": ("email",)}),
        (
            _("Permissions"),
            {
                "fields": (
                    "is_active",
                    "is_staff",
                    "is_superuser",
                    "groups",
                    "user_permissions",
                ),
                "classes": ["collapse"],
            },
        ),
    )
    list_display = ("username", "email", "is_staff", "hijack_field", "last_login")
    list_filter = ("is_staff", "is_superuser", "is_active", "groups")
    search_fields = ("username", "email")
    ordering = ("email",)
    readonly_fields = ("username", "last_login")
    inlines = [UserLegalAddressInline, UserProfileInline]

admin.site.unregister(User)
admin.site.register(User, UserAdmin)