from __future__ import annotations

import dataclasses
from typing import Optional, Text

import tensorflow as tf
import tensorflow_gnn as tfgnn

def _assert_rank1_int(t: tf.Tensor, tensor_name: Text) -> None:
  """Checks that the tensor `t` has dtype int32 or int64 with rank 1."""
  if t.shape.rank != 1 or t.dtype not in (tf.int32, tf.int64):
    raise ValueError(
        f'Expected `{tensor_name}` as rank-1 integer tensor,'
        f' got rank={t.shape.rank}, dtype={t.dtype.name}'
    )

def _row_lengths_to_row_ids(
    row_lengths: tf.Tensor, sum_row_lengths_hint: Optional[int] = None
) -> tf.Tensor:
  """Converts rank-1 ragged row lengths to row ids.

  For XLA compatibility `sum_row_lengths_hint` has to be provided to guarantee
  statically (compile-time) known result size.

  Example:

  ```python
  _row_lengths_to_row_ids([2, 1, 0, 2], 5)  # returns [0, 0, 1, 3, 3].
  ```

  Args:
    row_lengths: rank-1 integer tensor with ragged row lengths.
    sum_row_lengths_hint: value optionally provided by the client if the sum of
      `row_lengths` is known statically.

  Returns:
    Rank-1 integer tensor with ragged row ids.
  """
  _assert_rank1_int(row_lengths, 'row_lengths')

  row_starts = tf.math.cumsum(row_lengths)

  sum_row_lengths = (
      tf.reduce_sum(row_lengths)
      if sum_row_lengths_hint is None
      else sum_row_lengths_hint
  )

  cuts = tf.math.unsorted_segment_sum(
      tf.ones_like(row_starts), row_starts, sum_row_lengths + 1
  )
  result = tf.math.cumsum(cuts, exclusive=False)
  return result[:sum_row_lengths]

def _num_samples_per_segment(
    segment_sizes: tf.Tensor, max_samples: int
) -> tf.Tensor:
  """Returns a tensor representing counts to sample within each segment.

  Args:
    segment_sizes: Sizes tensors specifying counts within each segment.
    max_samples: Max number of entities to sample in total. Note that from each
      segment there'll be at most: `max_samples` / <segment_count> entities
      sampled based on the respective segment count or 1 entity per segment if
      the division `max_samples` / <segment_count> is below one. To improve the
      chances of samples returned from each segment, keep this number higher.

  Returns:
    Return a tensor with shape (num_segments,) that has the counts of elements
    to sample from each segment.
  """
  num_segments = segment_sizes.shape[0]
  segment_sample_count = max_samples // num_segments
  if segment_sample_count < 1:
    samples_per_segment = tf.ones_like(segment_sizes)
  else:
    segment_sample_count = tf.convert_to_tensor(
        [segment_sample_count], dtype=segment_sizes.dtype
    )
    samples_per_segment = tf.tile(
        segment_sample_count, tf.convert_to_tensor([num_segments])
    )

  return tf.math.minimum(samples_per_segment, segment_sizes)

def _find_different_indices(
    indices: tf.Tensor,
    another_indices: tf.Tensor,
    max_cols: int,
    default_value: int = -1,
) -> tf.Tensor:
  """Returns values(ids) from `indices` tensor that aren't in `another_indices`.

  Find the set difference between `indices` and `another_indices`
  and returns the difference in a dense matrix where columns are bounded by
  `max_cols` or the minimum number of identified differences in a row.

  Args:
    indices: A 2D matrix of indices.
    another_indices: Another 2D matrix of indices.
    max_cols: Maximum number of different elements to keep per row in the
      returned matrix.
    default_value: Default value to be used for dense tensor generation, it
      needs to be a value outside the vocabulary of either indices, and can be
      used as an oov id. Default is -1.

  Returns:
    Returns a dense tensor with shape [X, MIN(`max_cols`, Y)] where X is the
    first dimension of both `indices` and `another_indices`, while Y is
    the minimum row length of their set difference:
    (`indices` - `another_indices`).
  """
  diffed_indices = tf.sets.difference(indices, another_indices)
  diffed_indices = tf.sparse.to_dense(
      diffed_indices, default_value=default_value
  )
  ragged_diffed_indices = tf.RaggedTensor.from_tensor(
      diffed_indices, padding=default_value
  )
  row_count = indices.shape[0]
  min_index = int(tf.math.reduce_min(ragged_diffed_indices.row_lengths()))
  return ragged_diffed_indices.to_tensor(
      shape=[row_count, min(min_index, max_cols)]
  )

def segment_samples_to_indices(
    samples_per_segment: tf.Tensor,
    segment_sizes: tf.Tensor,
    *,
    seed: Optional[int] = None,
    sum_sample_sizes_hint: Optional[int] = None,
    sum_segment_sizes_hint: Optional[int] = None,
) -> tf.Tensor:
  """Returns sampled ids for each segment using counts in `samples_per_segment`.

  For XLA compatibility `sum_sample_sizes_hint` and `sum_segment_sizes_hint` has
  to be provided to guarantee statically (compile-time) known output size.

  Args:
    samples_per_segment: Number of samples to collect from each respective
      segment. This tensor's elements should be smaller than or equal to the
      `segment_sizes`.
    segment_sizes: Actual segment counts.
    seed: A random seed for node index shuffling.
    sum_sample_sizes_hint: Optional sizes value provided if the sum of the
      `samples_per_segment` known statically.
    sum_segment_sizes_hint: Optional sizes vlaue provided if the sum of the
      `segment_sizes` known statically.

  Returns:
    Tensor that lists ids sampled from a segment with sizes tensors passed
    in as `segment_sizes`. Returned tensor will have a certain number of ids
    sampled from each segment which is specified in `samples_per_segment`.
  """
  validation_ops = [
      tf.debugging.assert_shapes(
          [(segment_sizes, ('N',)), (samples_per_segment, ('N',))],
          message=(
              f'Tensor shapes for {segment_sizes} and '
              f'{samples_per_segment} has to match.'
          ),
      ),
      tf.debugging.assert_equal(
          tf.math.greater_equal(segment_sizes, samples_per_segment),
          tf.ones_like(segment_sizes, tf.bool),
          message=(
              f'Segment sizes in {segment_sizes} must be greater than or '
              f'equal to samples counts in {samples_per_segment}.'
          ),
      ),
  ]
  with tf.control_dependencies(validation_ops):
    samples_row_ids = _row_lengths_to_row_ids(
        samples_per_segment, sum_sample_sizes_hint
    )
    samples_segment_offsets = tf.cumsum(samples_per_segment, exclusive=True)
    samples_segment_offsets_per_index = tf.gather(
        samples_segment_offsets, samples_row_ids
    )
    # Find zero based indices within each segment.
    samples_segment_indices = tf.range(tf.math.reduce_sum(samples_per_segment))
    samples_segment_indices = tf.math.subtract(
        samples_segment_indices, samples_segment_offsets_per_index
    )

    full_segment_offsets = tf.cumsum(segment_sizes, exclusive=True)
    samples_full_segment_offsets_per_indice = tf.gather(
        full_segment_offsets, samples_row_ids
    )
    # Find actual node indices from full node ids, by adding the respective
    # segment row offsets to segment based indices
    samples_node_indices = tf.math.add(
        samples_segment_indices, samples_full_segment_offsets_per_indice
    )

    full_segment_row_ids = _row_lengths_to_row_ids(
        segment_sizes, sum_segment_sizes_hint
    )
    segment_shuffled_full_node_ids = (
        tfgnn.experimental.segment_random_index_shuffle(
            segment_ids=full_segment_row_ids, seed=seed
        )
    )
    subsampled_node_ids = tf.gather(
        segment_shuffled_full_node_ids, samples_node_indices
    )
    return subsampled_node_ids

@dataclasses.dataclass
class ConnectedNodes:
  """Keeps connected source and target node ids and node-set names.

  Attributes:
    source_node_name: Node-set name of the source endpoint.
    source_node_ids: Node ids from the source node-set.
    target_node_name: Node-set name of the target endpoint.
    target_node_ids: Node ids from the target node-set.
  """

  source_node_name: str
  source_node_ids: tf.Tensor
  target_node_name: str
  target_node_ids: tf.Tensor

def _get_connected_node_ids(
    graph: tfgnn.GraphTensor,
    *,
    edge_set_names: list[str],
    target_node_tag: tfgnn.IncidentNodeTag,
) -> ConnectedNodes:
  """Returns source and target node ids connected with edges in edge_set_names.

  Args:
    graph: A scalar GraphTensor.
    edge_set_names: Edge-set names for existing(positive) edge-set information.
      Each edge-set should have the same source and target node-set name.
    target_node_tag: Incident side of the edge-set adjacency representing target
      node set.

  Returns:
    A ConnectedNodes class which keeps source node ids, source node name and
    target node ids and target node name, that are connected with edges in
    `edge_set_names`. Target node-set is identified via the target_node_tag.

  Raises:
    ValueError: if graph is not scalar (rank > 0) or edge_set_names contains
    edge_sets with different source and target names.
  """
  tfgnn.check_scalar_graph_tensor(graph, '_get_connected_node_ids()')
  source_node_id_list = []
  target_node_id_list = []
  source_node_name = None
  target_node_name = None
  for edge_set_name in edge_set_names:
    edge_set = graph.edge_sets[edge_set_name]
    positive_adjacency = edge_set.adjacency
    if not source_node_name:
      source_node_name = positive_adjacency.source_name
    if source_node_name != positive_adjacency.source_name:
      raise ValueError(
          'source_name and target_name doesnt match among the edge_sets in '
          'the positive_edge_set_names, source_node_name: '
          f'{positive_adjacency.source_name} vs {source_node_name}'
      )
    if not target_node_name:
      target_node_name = positive_adjacency.target_name
    if target_node_name != positive_adjacency.target_name:
      raise ValueError(
          'source_name and target_name does not match among the'
          ' edge_sets in the positive_edge_set_names, '
          f'target_node_name: {positive_adjacency.target_name} vs '
          f'{target_node_name}'
      )
    source_node_id_list.append(positive_adjacency.source)
    target_node_id_list.append(positive_adjacency.target)
  if target_node_tag == tfgnn.TARGET:
    sampling_pos_source_node_ids = tf.concat(source_node_id_list, axis=0)
    sampling_pos_target_node_ids = tf.concat(target_node_id_list, axis=0)
  else:
    sampling_pos_source_node_ids = tf.concat(target_node_id_list, axis=0)
    sampling_pos_target_node_ids = tf.concat(source_node_id_list, axis=0)
  sampling_source_node_tag = tfgnn.reverse_tag(target_node_tag)
  sampling_source_node_name = positive_adjacency.node_set_name(
      sampling_source_node_tag
  )
  sampling_target_node_name = positive_adjacency.node_set_name(target_node_tag)
  return ConnectedNodes(
      sampling_source_node_name,
      sampling_pos_source_node_ids,
      sampling_target_node_name,
      sampling_pos_target_node_ids,
  )

def sample_unconnected_nodes(
    graph: tfgnn.GraphTensor,
    *,
    edge_set_names: list[str],
    max_negative_samples: int,
    negative_samples_node_tag: tfgnn.IncidentNodeTag,
    sample_buffer_scale: int = 2,
    seed: Optional[int] = None,
) -> tfgnn.Field:
  """Generates unconnected node-ids for the node-set of the specified tag.

  This operation returns up to `max_negative_samples` node-ids(sampling target
  node) from the node-set with the node-tag `negative_samples_node_tag` for each
  of the node-ids(sampling source) at the other end-point of the edge_sets
  specified by the `edge_set_names`. Returned node-ids indicate that there isn't
  an edge between the sampling target node-id and the respective sampling source
  node-id, also node-ids will be ordered by index starting with the smallest
  possible node-id. In case you prefer to shuffle the nodes within the graph
  tensor then please use  `shuffle_nodes()` before `sample_unconnected_nodes()`.

  Args:
    graph: A scalar GraphTensor.
    edge_set_names: Edge-set names for existing(positive) edge-set information.
      Each edge-set should have the same source and target node-set name.
    max_negative_samples: Maximum number of unconnected(negative) node-ids to
      return per node.
    negative_samples_node_tag: Incident side of the edge-set adjacency, among
      which unconnected(negative) node-ids will be generated.
    sample_buffer_scale: This scaling parameter will increase the total number
      of elements sampled per segment. Essentially we sample
      `max_negative_samples`*`sample_buffer_scale` elements in total.
    seed: Random seed parameter for shuffling node indices per segment before
      subsampling node-ids from each.

  Returns:
    Tensor specifying unconnected(negative) node-ids(sampling target) generated
    for the node-set identified with tag `negative_samples_node_tag` for each
    node(sampling source) at the other endpoint of the edge-set among the given
    `edge_set_names`. Returned tensor shape will be
    [X, MIN(`max_negative_samples`, Y)]; X equal to the total_size of the
    sampling source node-set, Y is the minimum of `max_negative_samples` and the
    minimum number of unconnected nodes across the sampling source node-set.
    Returned node-ids will be ordered at each row, starting at the smallest
    possible negative node-id.

  Raises:
    ValueError: if graph is not scalar (rank > 0) or positive_edge_set_names is
    empty.
  """
  tfgnn.check_scalar_graph_tensor(graph, 'sample_unconnected_nodes()')
  if not edge_set_names:
    raise ValueError('edge_set_names cant be empty')

  # Collect the list of source and target node ids which are connected by the
  # given edge_sets.
  sampling_connected_nodes = _get_connected_node_ids(
      graph,
      edge_set_names=edge_set_names,
      target_node_tag=negative_samples_node_tag,
  )

  total_sampling_sources = graph.node_sets[
      sampling_connected_nodes.source_node_name
  ].total_size
  sampling_target_node_sizes = graph.node_sets[
      sampling_connected_nodes.target_node_name
  ].sizes
  sampling_target_node_sizes_hint = graph.node_sets[
      sampling_connected_nodes.target_node_name
  ].spec.total_size

  # Generate sample count to collect from each component.
  segment_samples_count = _num_samples_per_segment(
      sampling_target_node_sizes, max_negative_samples * sample_buffer_scale
  )
  subsampled_negative_node_ids = segment_samples_to_indices(
      segment_samples_count,
      sampling_target_node_sizes,
      seed=seed,
      sum_sample_sizes_hint=tf.reduce_sum(segment_samples_count),
      sum_segment_sizes_hint=sampling_target_node_sizes_hint,
  )
  # Tile negative_node_ids to represent the potential negative nodes for each
  # sampling source node (SSN).
  # [0 1 ... N, 0 1 ... N, ... 0 1 ... N]
  # < SSN_1   > <  SSN_2 > ... <  SSN_X >
  # Tile to represent each sampling source node in their respective row.
  subsampled_negative_node_ids = tf.tile(
      tf.expand_dims(subsampled_negative_node_ids, axis=0),
      [total_sampling_sources, 1],
  )
  # Get positive(existing edge-sets) sampling target node-ids for each
  # sampling source node.
  sampling_pos_source_node_sorted_indices = tf.argsort(
      sampling_connected_nodes.source_node_ids
  )
  sampling_pos_source_node_ids = tf.gather(
      sampling_connected_nodes.source_node_ids,
      sampling_pos_source_node_sorted_indices,
  )
  sampling_pos_target_node_ids = tf.gather(
      sampling_connected_nodes.target_node_ids,
      sampling_pos_source_node_sorted_indices,
  )
  positive_target_node_ids = tf.RaggedTensor.from_value_rowids(
      sampling_pos_target_node_ids,
      sampling_pos_source_node_ids,
      nrows=total_sampling_sources,
  )
  positive_target_node_ids = positive_target_node_ids.to_tensor(
      default_value=-1
  )
  unconnected_node_ids = _find_different_indices(
      subsampled_negative_node_ids,
      positive_target_node_ids,
      max_negative_samples,
  )
  return unconnected_node_ids


#! /usr/bin/env python

# Copyright 2011 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Merges the results of processJavaScriptSource.py and processHTMLSources.py
# into a single TS file that can be translated and loaded by BumpTop

import json
import os
import sys
import localization # BumpTop helper module for localization

# Take a path relative to this script, return a path relative to the working dir
def script_relative(path):
	thisdir = os.path.dirname(os.path.realpath(__file__))
	return os.path.normpath(os.path.join(thisdir, path))

SOURCES = [
	script_relative("../Languages/BumpTopJavaScript_en.json"),
	script_relative("../Languages/BumpTopJavaHTML_en.json")
]

QT_TRANSLATIONS_FILENAME = script_relative("../Languages/BumpTop_Web_en.ts")
translation_entries = []
existing_translation_entry_sources = set()

SCRIPT_NAME = os.path.basename(__file__)
logger = localization.Logger(SCRIPT_NAME)

# Read all the translation entries
for filename in SOURCES:
	fd = open(filename, 'r')
	logger.info("Processing %s..." % filename)
	data = json.load(fd)
	for entry in data:
		# ensure that there are no duplicate sources
		if entry["source"] not in existing_translation_entry_sources:
			translation_entries.append(entry)
			existing_translation_entry_sources.add(entry["source"])
	fd.close()

# Helper func to html-encode a string
htmlCodes = [
    ['&', '&'],
    ['<', '<'],
    ['>', '>'],
    ['"', '"'],
]
def htmlEncode(s):
    for code in htmlCodes:
        s = s.replace(code[0], code[1])
    return s

# Write the TS file output
fd = open(QT_TRANSLATIONS_FILENAME, 'w+')
fd.write(
'''<?xml version="1.0" encoding="utf-8"?>
<!--
// Copyright 2011 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
-->
<!DOCTYPE TS>
<TS version="2.0" language="en_US">
<context>
    <name>JavaScriptAPI</name>
''')
for entry in translation_entries:
	fd.write(
	'''
	<message>
        <location filename="%s" line="0"/>
        <source>%s</source>
        <translation>%s</translation>
    </message>''' % (htmlEncode(entry['location']), htmlEncode(entry['source']), htmlEncode(entry['translation'])))

fd.write(
'''</context>
</TS>
''')
fd.close()


from os.path import dirname, realpath, sep, pardir
import sys
import glob
sys.path.insert(0, dirname(realpath(__name__)) + sep + pardir + sep + "CellCNN")

from Dataset import DataDataset, DatasetSplit
from InputData import InputData
from Models import CellCNN
import matplotlib.pyplot as plt
# CSV data is in '../csv/*.csv'
# labels are in '../summary.csv'
CSV_FOLDER = "csv\\"
SUMMARY_FILE = "summary.csv"
CODE_NAME = "Kod"

CATEGORICAL = ["Pacient_._Kontrola",
                "Od_koho",
                "sex",
                "Subjective_evaluation_of_the_physitian"]
# NUMERICAL = ["Age",
#             "Age_of_onset",
#             "Length_of_the_disease",
#             "Age_at_diagnosis",
#             "Years_on_Ig_treatment",
#             "Dg_Delay",
#             "IgG_at_Dg",
#             "IgA_at_Dg",
#             "IgM_at_Dg",
#             "obins_hvs5",
#             "ubins_hvs5",
#             "ubins1",
#             "ubins2"]
NUMERICAL = [
            "Length_of_the_disease",
            "IgG_at_Dg",
            "IgA_at_Dg",
            "IgM_at_Dg",
            "obins_hvs5",
            "ubins_hvs5"]

import tensorflow as tf
import pandas as pd
import numpy as np

def get_summary_file()->pd.DataFrame:
    summary = pd.read_csv(SUMMARY_FILE)
    summary = summary.replace("?", -1)
    summary = summary.fillna(-1)
    summary[CATEGORICAL] = summary[CATEGORICAL].astype("category")
    summary[NUMERICAL] = summary[NUMERICAL].apply(pd.to_numeric, errors="coerce")
    return summary
print("Getting summary")
summary = get_summary_file()
data = []
labels = []
print("reading data")
for i in range(len(summary)):
    code = summary.loc[i, CODE_NAME]
    for file_name in glob.glob(CSV_FOLDER+code+"*.csv"):
        panda = pd.read_csv(file_name).drop("Unnamed: 0", 1)
        data.append(DataDataset(panda.to_numpy())) # Ignore cell numbering
        panda_label = summary.loc[i, NUMERICAL]
        labels.append(panda_label.to_numpy())
print("Generating inputs")
labels = np.array(labels)
inp = InputData(data, labels=labels)
X,Y = inp.get_multi_cell_inputs(2500, DatasetSplit.TRAIN)
Xt,Yt = inp.get_multi_cell_inputs(500, DatasetSplit.TEST)
print("Doing stuff with Model")
model = CellCNN((None, 1000, 20), [len(NUMERICAL),], conv=[16,],)
callb = model.fit(X,Y, epochs=5, validation_data=(Xt,Yt))
plt.plot(callb.history["val_loss"])
plt.show()
s = model.get_single_cell_model()
for i in range(20):
    for j in range(i+1, 20):
        s.show_importance(X[0], scale=True, filters = (1,), dimensions = (i,j)) # This shows values only for first 2 dimensions!


from typing import Dict

from torch import Tensor
from torch.types import Device

def dict_to_device(d: Dict, device: Device) -> Dict | None:
    if d is None:
        return None
    for k, v in d.items():
        if isinstance(v, Tensor):
            d[k] = d[k].to(device)
    return d


import simplegui
char_v=" "

def hmm(canvas):
    canvas.draw_text(char_v,[1,99],99,"white")

def k_d(key):
    global char_v
    char_v=chr(key)

def k_u(key):
    global char_v
    char_v=" "

frame1=simplegui.create_frame("key test",100,100)
frame1.set_draw_handler(hmm)
frame1.set_keydown_handler(k_d)
frame1.set_keyup_handler(k_u)
frame1.start()


import logging
import os
from future.backports.urllib.parse import parse_qs
from future.backports.urllib.parse import urlparse

from oic.utils.http_util import extract_from_request
from oic.utils.http_util import Response
from oic.utils.http_util import Unauthorized
from oic.utils.http_util import NotFound
from oic.utils.http_util import ServiceError
from oic.utils.http_util import BadRequest
from oic.utils.webfinger import OIC_ISSUER
from oic.utils.webfinger import WebFinger

from otest import pathmap

__author__ = 'roland'

LOGGER = logging.getLogger(__name__)

HEADER = "---------- %s ----------"

def dump_log(session_info, trace):
    file_name = os.path.join("log", session_info["addr"],
                             session_info["test_id"])
    try:
        fp = open(file_name, "a+")
    except IOError:
        try:
            os.makedirs(os.path.join("log", session_info["addr"]))
        except OSError:
            pass

        try:
            fp = open(file_name, "w")
        except Exception as err:
            logging.error(
                "Couldn't dump to log file {} reason: {}").format(
                file_name, err)
            raise

    fp.write("{0}".format(trace))
    fp.write("\n\n")
    fp.close()

def find_identifier(uri):
    if uri.startswith("http"):
        p = urlparse(uri)
        return p.path[1:]  # Skip leading "/"
    elif uri.startswith("acct:"):
        a = uri[5:]
        l, d = a.split("@")
        return l

def wsgi_wrapper(environ, start_response, func, session_info, trace, **kwargs):
    kwargs = extract_from_request(environ)
    trace.request(kwargs["request"])
    args = func(**kwargs)

    try:
        resp, state = args
        trace.reply(resp.message)
        dump_log(session_info, trace)
        return resp(environ, start_response)
    except TypeError:
        resp = args
        trace.reply(resp.message)
        dump_log(session_info, trace)
        return resp(environ, start_response)
    except Exception as err:
        # LOGGER.error("%s" % err)
        trace.error("%s" % err)
        dump_log(session_info, trace)
        raise

# noinspection PyUnusedLocal
def token(environ, start_response, session_info, trace, **kwargs):
    trace.info(HEADER % "AccessToken")
    _op = session_info["op"]

    return wsgi_wrapper(environ, start_response, _op.token_endpoint,
                        session_info,
                        trace)

# noinspection PyUnusedLocal
def authorization(environ, start_response, session_info, trace, **kwargs):
    trace.info(HEADER % "Authorization")
    _op = session_info["op"]

    return wsgi_wrapper(environ, start_response, _op.authorization_endpoint,
                        session_info, trace)

# noinspection PyUnusedLocal
def userinfo(environ, start_response, session_info, trace, **kwargs):
    trace.info(HEADER % "UserInfo")
    _op = session_info["op"]
    return wsgi_wrapper(environ, start_response, _op.userinfo_endpoint,
                        session_info, trace)

# noinspection PyUnusedLocal
def op_info(environ, start_response, session_info, trace, **kwargs):
    trace.info(HEADER % "ProviderConfiguration")
    trace.request("PATH: %s" % environ["PATH_INFO"])
    try:
        trace.request("QUERY: %s" % environ["QUERY_STRING"])
    except KeyError:
        pass
    _op = session_info["op"]
    return wsgi_wrapper(environ, start_response, _op.providerinfo_endpoint,
                        session_info, trace)

# noinspection PyUnusedLocal
def registration(environ, start_response, session_info, trace, **kwargs):
    trace.info(HEADER % "ClientRegistration")
    _op = session_info["op"]

    if environ["REQUEST_METHOD"] == "POST":
        return wsgi_wrapper(environ, start_response, _op.registration_endpoint,
                            session_info, trace)
    elif environ["REQUEST_METHOD"] == "GET":
        return wsgi_wrapper(environ, start_response, _op.read_registration,
                            session_info, trace)
    else:
        resp = ServiceError("Method not supported")
        return resp(environ, start_response)

# noinspection PyUnusedLocal
def check_id(environ, start_response, session_info, trace, **kwargs):
    _op = session_info["op"]

    return wsgi_wrapper(environ, start_response, _op.check_id_endpoint,
                        session_info, trace)

# noinspection PyUnusedLocal
def endsession(environ, start_response, session_info, trace, **kwargs):
    _op = session_info["op"]
    return wsgi_wrapper(environ, start_response, _op.endsession_endpoint,
                        session_info=session_info, trace=trace)

def webfinger(environ, start_response, session_info, trace, **kwargs):
    query = parse_qs(environ["QUERY_STRING"])

    # Find the identifier
    session_info["test_id"] = find_identifier(query["resource"][0])

    trace.info(HEADER % "WebFinger")
    trace.request(environ["QUERY_STRING"])
    trace.info("QUERY: %s" % (query,))

    try:
        assert query["rel"] == [OIC_ISSUER]
        resource = query["resource"][0]
    except AssertionError:
        errmsg = "Wrong 'rel' value: %s" % query["rel"][0]
        trace.error(errmsg)
        resp = BadRequest(errmsg)
    except KeyError:
        errmsg = "Missing 'rel' parameter in request"
        trace.error(errmsg)
        resp = BadRequest(errmsg)
    else:
        wf = WebFinger()
        p = urlparse(resource)

        if p.scheme == "acct":
            l, _ = p.path.split("@")
            path = pathmap.IDMAP[l.lower()]
        else:  # scheme == http/-s
            try:
                path = pathmap.IDMAP[p.path[1:].lower()]
            except KeyError:
                path = None

        if path:
            _url = os.path.join(kwargs["op_arg"]["baseurl"],
                                session_info["test_id"],
                                path[1:])
            resp = Response(wf.response(subject=resource, base=_url),
                            content="application/jrd+json")
        else:
            resp = BadRequest("Incorrect resource specification")

        trace.reply(resp.message)

    dump_log(session_info, trace)
    return resp(environ, start_response)

# noinspection PyUnusedLocal
def verify(environ, start_response, session_info, trace, **kwargs):
    _op = session_info["op"]
    return wsgi_wrapper(environ, start_response, _op.verify_endpoint,
                        session_info, trace)

def static_file(path):
    try:
        os.stat(path)
        return True
    except OSError:
        return False

# noinspection PyUnresolvedReferences
def static(environ, start_response, path):
    LOGGER.info("[static]sending: %s" % (path,))

    try:
        text = open(path).read()
        if path.endswith(".ico"):
            start_response('200 OK', [('Content-Type', "image/x-icon")])
        elif path.endswith(".html"):
            start_response('200 OK', [('Content-Type', 'text/html')])
        elif path.endswith(".json"):
            start_response('200 OK', [('Content-Type', 'application/json')])
        elif path.endswith(".txt"):
            start_response('200 OK', [('Content-Type', 'text/plain')])
        elif path.endswith(".css"):
            start_response('200 OK', [('Content-Type', 'text/css')])
        else:
            start_response('200 OK', [('Content-Type', 'text/plain')])
        return [text]
    except IOError:
        resp = NotFound()
        return resp(environ, start_response)

# noinspection PyUnusedLocal
def safe(environ, start_response):
    _op = environ["oic.oas"]
    _srv = _op.server
    _log_info = _op.logger.info

    _log_info("- safe -")
    # _log_info("env: %s" % environ)
    # _log_info("handle: %s" % (handle,))

    try:
        _authz = environ["HTTP_AUTHORIZATION"]
        (_typ, code) = _authz.split(" ")
        assert _typ == "Bearer"
    except KeyError:
        resp = BadRequest("Missing authorization information")
        return resp(environ, start_response)

    try:
        _sinfo = _srv.sdb[code]
    except KeyError:
        resp = Unauthorized("Not authorized")
        return resp(environ, start_response)

    _info = "'%s' secrets" % _sinfo["sub"]
    resp = Response(_info)
    return resp(environ, start_response)

# noinspection PyUnusedLocal
def css(environ, start_response):
    try:
        _info = open(environ["PATH_INFO"]).read()
        resp = Response(_info)
    except (OSError, IOError):
        resp = NotFound(environ["PATH_INFO"])

    return resp(environ, start_response)

# ----------------------------------------------------------------------------

def display_log(environ, start_response, lookup):
    path = environ.get('PATH_INFO', '').lstrip('/')
    if path == "log":
        tail = environ["REMOTE_ADDR"]
        path = os.path.join(path, tail)
    elif path == "logs":
        path = "log"

    if os.path.isfile(path):
        return static(environ, start_response, path)
    elif os.path.isdir(path):
        item = []
        for (dirpath, dirnames, filenames) in os.walk(path):
            if dirnames:
                # item = [(fn, os.path.join(path, fn)) for fn in dirnames]
                item = [(fn, fn) for fn in dirnames]
                break
            if filenames:
                # item = [(fn, os.path.join(path, fn)) for fn in filenames]
                item = [(fn, fn) for fn in filenames]
                break

        item.sort()
        resp = Response(mako_template="logs.mako",
                        template_lookup=lookup,
                        headers=[])
        argv = {"logs": item}

        return resp(environ, start_response, **argv)
    else:
        resp = Response("No saved logs")
        return resp(environ, start_response)


#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import csv
import json
import os
import re
import shutil
import subprocess
import sys

from talos.xtalos import xtalos

EVENTNAME_INDEX = 0
PROCESS_INDEX = 2
THREAD_ID_INDEX = 3
PARENT_PID_INDEX = 3
DISKBYTES_COL = "Size"
FNAME_COL = "FileName"
IMAGEFUNC_COL = "Image!Function"
EVENTGUID_COL = "EventGuid"
ACTIVITY_ID_COL = "etw:ActivityId"
NUMBYTES_COL = "NumBytes"
BYTESSENT_COL = "BytesSent"

CEVT_WINDOWS_RESTORED = "{917b96b1-ecad-4dab-a760-8d49027748ae}"
CEVT_XPCOM_SHUTDOWN = "{26d1e091-0ae7-4f49-a554-4214445c505c}"
NAME_SUBSTITUTIONS = [
    # Careful with your regex!
    # Substitution happens after combinations like \t \s \n ... are replaced
    # with their real representations. So, prepend them with extra backslash.
    # Read more: https://docs.python.org/2.7/library/re.html#re.sub
    (re.compile(r"{\w{8}-\w{4}-\w{4}-\w{4}-\w{12}}"), "{uuid}"),
    (re.compile(r"talos\\tests\\tp5n\\.*"), r"talos\\tests\{tp5n_files}"),
    (re.compile(r"nvidia corporation\\3d vision\\.*"), "{nvidia_3d_vision}"),
    (re.compile(r"cltbld\.t-w732-ix-\d+\.\d+"), "{cltbld}"),
    (
        re.compile(r"venv\\lib\\site-packages\\pip\\_vendor\\.*"),
        r"venv\\lib\\site-packages\\{pip_vendor}",
    ),
]
stages = ["startup", "normal", "shutdown"]
net_events = {
    "TcpDataTransferReceive": "recv",
    "UdpEndpointReceiveMessages": "recv",
    "TcpDataTransferSend": "send",
    "UdpEndpointSendMessages": "send",
}
gThreads = {}
gConnectionIDs = {}
gHeaders = {}
gBrowserPID = None

def uploadFile(filename):
    mud = os.environ.get("MOZ_UPLOAD_DIR", None)
    if mud:
        print("uploading raw file %s via blobber" % filename)
        mud_filename = os.path.join(mud, filename)
        shutil.copyfile(filename, "%s.log" % mud_filename)

def filterOutHeader(data):
    # -1 means we have not yet found the header
    # 0 means we are in the header
    # 1+ means that we are past the header
    state = -1
    done = False
    while not done:
        try:
            row = next(data)
        except StopIteration:
            done = True
            break
        except UnicodeDecodeError:
            continue
        except csv.Error:
            continue

        if not len(row):
            continue

        if state < 0:
            # Keep looking for the header (denoted by "BeginHeader").
            if row[0] == "BeginHeader":
                state = 0
            continue

        if state == 0:
            # Eventually, we'll find the end (denoted by "EndHeader").
            if row[0] == "EndHeader":
                state = 1
                continue

            gHeaders[row[EVENTNAME_INDEX]] = row
            continue

        if state >= 1:
            state = state + 1

        # The line after "EndHeader" is also not useful, so we want to strip
        # that in addition to the header.
        if state > 2:
            yield row

def getIndex(eventName, *colNames):
    eventHeader = gHeaders[eventName]

    for colName in colNames:
        try:
            return eventHeader.index(colName)
        except ValueError:
            pass

    return None

def readFile(filename):
    data = csv.reader(
        open(filename, "r", encoding="cp1252"),
        delimiter=",",
        quotechar='"',
        skipinitialspace=True,
    )
    data = filterOutHeader(data)
    return data

def fileSummary(row, stage, retVal):
    event = row[EVENTNAME_INDEX]

    # TODO: do we care about the other events?
    if event not in ("FileIoRead", "FileIoWrite"):
        return
    fname_index = getIndex(event, FNAME_COL)

    # We only care about events that have a file name.
    if fname_index is None:
        return

    # Some data rows are missing the filename?
    if len(row) <= fname_index:
        return

    thread_extra = ""
    if gThreads[row[THREAD_ID_INDEX]] == "main":
        thread_extra = " (main)"
    key_tuple = (
        row[fname_index],
        "%s%s" % (row[THREAD_ID_INDEX], thread_extra),
        stages[stage],
    )
    total_tuple = (
        row[fname_index],
        "%s%s" % (row[THREAD_ID_INDEX], thread_extra),
        "all",
    )

    if key_tuple not in retVal:
        retVal[key_tuple] = {
            "DiskReadBytes": 0,
            "DiskReadCount": 0,
            "DiskWriteBytes": 0,
            "DiskWriteCount": 0,
        }
        retVal[total_tuple] = retVal[key_tuple]

    if event == "FileIoRead":
        retVal[key_tuple]["DiskReadCount"] += 1
        retVal[total_tuple]["DiskReadCount"] += 1
        idx = getIndex(event, DISKBYTES_COL)
        retVal[key_tuple]["DiskReadBytes"] += int(row[idx], 16)
        retVal[total_tuple]["DiskReadBytes"] += int(row[idx], 16)
    elif event == "FileIoWrite":
        retVal[key_tuple]["DiskWriteCount"] += 1
        retVal[total_tuple]["DiskWriteCount"] += 1
        idx = getIndex(event, DISKBYTES_COL)
        retVal[key_tuple]["DiskWriteBytes"] += int(row[idx], 16)
        retVal[total_tuple]["DiskWriteBytes"] += int(row[idx], 16)

def etl2csv(xperf_path, etl_filename, debug=False):
    """
    Convert etl_filename to etl_filename.csv (temp file) which is the .csv
    representation of the .etl file Etlparser will read this .csv and parse
    the information we care about into the final output. This is done to keep
    things simple and to preserve resources on talos machines
    (large files == high memory + cpu)
    """

    xperf_cmd = [
        xperf_path,
        "-merge",
        "%s.user" % etl_filename,
        "%s.kernel" % etl_filename,
        etl_filename,
    ]
    if debug:
        print("executing '%s'" % subprocess.list2cmdline(xperf_cmd))
    subprocess.call(xperf_cmd)

    csv_filename = "%s.csv" % etl_filename
    xperf_cmd = [xperf_path, "-i", etl_filename, "-o", csv_filename]
    if debug:
        print("executing '%s'" % subprocess.list2cmdline(xperf_cmd))
    subprocess.call(xperf_cmd)
    return csv_filename

def trackProcess(row, firstFirefoxPID):
    global gBrowserPID
    if gBrowserPID:
        return

    # Without the launcher, the initial Firefox process *is* the browser
    # process. OTOH, with the launcher process enabled, the browser is actually
    # the first child process of the first Firefox process.
    parentPID = int(row[PARENT_PID_INDEX])
    if parentPID == firstFirefoxPID:
        proc = row[PROCESS_INDEX]
        gBrowserPID = int(re.search("^.* \(\s*(\d+)\)$", proc).group(1))

def getBrowserPID():
    global gBrowserPID
    return gBrowserPID

def trackThread(row, browserPID):
    event, proc, tid = row[EVENTNAME_INDEX], row[PROCESS_INDEX], row[THREAD_ID_INDEX]
    if event in ["T-DCStart", "T-Start"]:
        procName, procID = re.search("^(.*) \(\s*(\d+)\)$", proc).group(1, 2)
        if procID == str(browserPID):
            imgIdx = getIndex(event, IMAGEFUNC_COL)
            img = re.match("([^!]+)!", row[imgIdx]).group(1)
            if img == procName:
                gThreads[tid] = "main"
            else:
                gThreads[tid] = "nonmain"
    elif event in ["T-DCEnd", "T-End"] and tid in gThreads:
        del gThreads[tid]

def trackThreadFileIO(row, io, stage):
    event, tid = row[EVENTNAME_INDEX], row[THREAD_ID_INDEX]
    opType = {"FileIoWrite": "write", "FileIoRead": "read"}[event]
    th, stg = gThreads[tid], stages[stage]
    sizeIdx = getIndex(event, DISKBYTES_COL)
    bytes = int(row[sizeIdx], 16)
    io[(th, stg, "file_%s_ops" % opType)] = (
        io.get((th, stg, "file_%s_ops" % opType), 0) + 1
    )
    io[(th, stg, "file_%s_bytes" % opType)] = (
        io.get((th, stg, "file_%s_bytes" % opType), 0) + bytes
    )
    io[(th, stg, "file_io_bytes")] = io.get((th, stg, "file_io_bytes"), 0) + bytes

def trackThreadNetIO(row, io, stage):
    event, tid = row[EVENTNAME_INDEX], row[THREAD_ID_INDEX]
    connIdIdx = getIndex(event, ACTIVITY_ID_COL)
    connID = row[connIdIdx]
    if connID not in gConnectionIDs:
        gConnectionIDs[connID] = tid
    origThread = gConnectionIDs[connID]
    if origThread in gThreads:
        match = re.match("[\w-]+\/([\w-]+)?", event)
        if not match:
            raise xtalos.XTalosError(
                "Could not find a regular expression match for event: {}".format(event)
            )
        netEvt = match.group(1)

        if netEvt in net_events:
            opType = net_events[netEvt]
            th, stg = gThreads[origThread], stages[stage]
            # On newer versions of Windows, some net I/O events have switched to
            # using BYTESSENT_COL, so we try both
            lenIdx = getIndex(event, NUMBYTES_COL, BYTESSENT_COL)
            bytes = int(row[lenIdx])
            io[(th, stg, "net_%s_bytes" % opType)] = (
                io.get((th, stg, "net_%s_bytes" % opType), 0) + bytes
            )
            io[(th, stg, "net_io_bytes")] = io.get((th, stg, "net_io_bytes"), 0) + bytes

def updateStage(row, stage):
    guidIdx = getIndex(row[EVENTNAME_INDEX], EVENTGUID_COL)
    if row[guidIdx] == CEVT_WINDOWS_RESTORED and stage == 0:
        stage = 1
    elif row[guidIdx] == CEVT_XPCOM_SHUTDOWN and stage == 1:
        stage = 2
    return stage

def loadAllowlist(filename):
    if not filename:
        return
    if not os.path.exists(filename):
        print("Warning: xperf allowlist %s was not found" % filename)
        return
    lines = open(filename).readlines()
    # Expand paths
    lines = [os.path.expandvars(elem.strip()) for elem in lines]
    files = set()
    dirs = set()
    recur = set()
    for line in lines:
        if line.startswith("#"):
            continue
        elif line.endswith("\\*\\*"):
            recur.add(line[:-4])
        elif line.endswith("\\*"):
            dirs.add(line[:-2])
        else:
            files.add(line)
    return (files, dirs, recur)

def checkAllowlist(filename, allowlist):
    if not allowlist:
        return False
    if filename in allowlist[0]:
        return True
    if os.path.dirname(filename) in allowlist[1]:
        return True
    head = filename
    while len(head) > 3:  # Length 3 implies root directory, e.g. C:\
        head, tail = os.path.split(head)
        if head in allowlist[2]:
            return True
    return False

def etlparser(
    xperf_path,
    etl_filename,
    processID,
    approot=None,
    configFile=None,
    outputFile=None,
    allowlist_file=None,
    error_filename=None,
    all_stages=False,
    all_threads=False,
    debug=False,
):
    global NAME_SUBSTITUTIONS

    # setup output file
    if outputFile:
        outFile = open(outputFile, "w")
    else:
        outFile = sys.stdout

    files = {}
    io = {}
    stage = 0

    csvname = etl2csv(xperf_path, etl_filename, debug=debug)
    for row in readFile(csvname):
        event = row[EVENTNAME_INDEX]
        if event == "P-Start":
            trackProcess(row, processID)
        elif event in ["T-DCStart", "T-Start", "T-DCEnd", "T-End"]:
            trackThread(row, getBrowserPID())
        elif (
            event in ["FileIoRead", "FileIoWrite"] and row[THREAD_ID_INDEX] in gThreads
        ):
            fileSummary(row, stage, files)
            trackThreadFileIO(row, io, stage)
        elif event.endswith("Event/Classic") and row[THREAD_ID_INDEX] in gThreads:
            stage = updateStage(row, stage)
        elif event.startswith("Microsoft-Windows-TCPIP"):
            trackThreadNetIO(row, io, stage)

    if debug:
        uploadFile(csvname)

    output = "thread, stage, counter, value\n"
    for cntr in sorted(io.keys()):
        output += "%s, %s\n" % (", ".join(cntr), str(io[cntr]))
    if outputFile:
        fname = "%s_thread_stats%s" % os.path.splitext(outputFile)
        with open(fname, "w") as f:
            f.write(output)

        if debug:
            uploadFile(fname)
    else:
        print(output)

    allowlist = loadAllowlist(allowlist_file)

    header = "filename, tid, stage, readcount, readbytes, writecount," " writebytes"
    outFile.write(header + "\n")

    # Filter out stages, threads, and allowlisted files that we're not
    # interested in
    filekeys = [
        x
        for x in files.keys()
        if (all_stages or x[2] == stages[0])
        and (all_threads or x[1].endswith("(main)"))
        and (all_stages and x[2] != stages[0] or not checkAllowlist(x[0], allowlist))
    ]
    if debug:
        # in debug, we want stages = [startup+normal] and all threads, not just (main)
        # we will use this data to upload fileIO info to blobber only for debug mode
        outputData = [
            x
            for x in files.keys()
            if (all_stages or x[2] in [stages[0], stages[1]])
            and (
                all_stages
                and x[2] not in [stages[0], stages[1]]
                or not checkAllowlist(x[0], allowlist)
            )
        ]
    else:
        outputData = filekeys

    # output data
    for row in outputData:
        output = "%s, %s, %s, %s, %s, %s, %s\n" % (
            row[0],
            row[1],
            row[2],
            files[row]["DiskReadCount"],
            files[row]["DiskReadBytes"],
            files[row]["DiskWriteCount"],
            files[row]["DiskWriteBytes"],
        )
        outFile.write(output)

    if outputFile:
        # close the file handle
        outFile.close()
        if debug:
            uploadFile(outputFile)

    # We still like to have the outputfile to record the raw data, now
    # filter out acceptable files/ranges
    allowlist_path = None
    wl_temp = {}
    dirname = os.path.dirname(__file__)
    if os.path.exists(os.path.join(dirname, "xperf_allowlist.json")):
        allowlist_path = os.path.join(dirname, "xperf_allowlist.json")
    elif os.path.exists(os.path.join(dirname, "xtalos")) and os.path.exists(
        os.path.join(dirname, "xtalos", "xperf_allowlist.json")
    ):
        allowlist_path = os.path.join(dirname, "xtalos", "xperf_allowlist.json")

    wl_temp = {}
    if allowlist_path:
        with open(allowlist_path, "r") as fHandle:
            wl_temp = json.load(fHandle)

    # Approot is the full path where the application is located at
    # We depend on it for dependentlibs.list to ignore files required for
    # normal startup.
    if approot:
        if os.path.exists("%s\\dependentlibs.list" % approot):
            with open("%s\\dependentlibs.list" % approot, "r") as fhandle:
                libs = fhandle.readlines()

            for lib in libs:
                wl_temp["{firefox}\\%s" % lib.strip()] = {"ignore": True}

    # Windows isn't case sensitive, this protects us against mismatched
    # systems.
    wl = {}
    for item in wl_temp:
        wl[item.lower()] = wl_temp[item]

    errors = []
    for row in filekeys:
        filename = original_filename = row[0]
        filename = filename.lower()
        # take care of 'program files (x86)' matching 'program files'
        filename = filename.replace(" (x86)", "")

        paths = ["profile", "firefox", "desktop", "talos"]
        for path in paths:
            pathname = "%s\\" % path
            parts = filename.split(pathname)
            if len(parts) >= 2:
                filename = "{%s}\\%s" % (path, pathname.join(parts[1:]))

        parts = filename.split("\\installtime")
        if len(parts) >= 2:
            filename = "%s\\{time}" % parts[0]

        # NOTE: this is Prefetch or prefetch, not case sensitive operating
        # system
        parts = filename.split("refetch")
        if len(parts) >= 2:
            filename = "%srefetch\\{prefetch}.pf" % parts[0]

        for pattern, substitution in NAME_SUBSTITUTIONS:
            filename = re.sub(pattern, substitution, filename)

        if filename in wl:
            if "ignore" in wl[filename] and wl[filename]["ignore"]:
                continue

        # too noisy
        #            if wl[filename]['minbytes'] > (files[row]['DiskReadBytes'] +\
        #                   files[row]['DiskWriteBytes']):
        #                print "%s: read %s bytes, less than expected minimum: %s"
        #                       % (filename, (files[row]['DiskReadBytes'] +
        #                                     files[row]['DiskWriteBytes']),
        #                          wl[filename]['minbytes'])

        # don't report in first round
        #            elif wl[filename]['maxbytes'] < (files[row]['DiskReadBytes'] +\
        #                   files[row]['DiskWriteBytes']):
        #                errors.append("%s: read %s bytes, more than expected maximum:"
        #                              " %s"
        #                              % (filename, (files[row]['DiskReadBytes'] +
        #                                 files[row]['DiskWriteBytes']),
        #                              wl[filename]['maxbytes']))

        # too noisy
        #            elif wl[filename]['mincount'] > (files[row]['DiskReadCount'] +\
        #                   files[row]['DiskWriteCount']):
        #                print "%s: %s accesses, less than expected minimum: %s"
        #                       % (filename, (files[row]['DiskReadCount'] +
        #                                     files[row]['DiskWriteCount']),
        #                          wl[filename]['mincount'])

        # don't report in first round
        #            elif wl[filename]['maxcount'] < (files[row]['DiskReadCount'] +\
        #                   files[row]['DiskWriteCount']):
        #                errors.append("%s: %s accesses, more than expected maximum:"
        #                              " %s"
        #                              % (filename, (files[row]['DiskReadCount'] +
        #                                            files[row]['DiskWriteCount']),
        #                                 wl[filename]['maxcount']))
        else:
            errors.append(
                "File '%s' (normalized from '%s') was accessed and we were not expecting"
                " it.  DiskReadCount: %s, DiskWriteCount: %s,"
                " DiskReadBytes: %s, DiskWriteBytes: %s"
                % (
                    filename,
                    original_filename,
                    files[row]["DiskReadCount"],
                    files[row]["DiskWriteCount"],
                    files[row]["DiskReadBytes"],
                    files[row]["DiskWriteBytes"],
                )
            )

    if errors:
        # output specific errors to be picked up by tbpl parser
        for error in errors:
            # NOTE: the ' :' is intentional, without the space before the :,
            # some parser will translate this
            print("TEST-UNEXPECTED-FAIL : xperf: %s" % error)

        # We detect if browser_failures.txt exists to exit and turn the job
        # orange
        if error_filename:
            with open(error_filename, "w") as errorFile:
                errorFile.write("\n".join(errors))

        if debug:
            uploadFile(etl_filename)

def etlparser_from_config(config_file, **kwargs):
    """start from a YAML config file"""

    # option defaults
    args = {
        "xperf_path": "xperf.exe",
        "etl_filename": "test.etl",
        "outputFile": "etl_output.csv",
        "processID": None,
        "approot": None,
        "allowlist_file": None,
        "error_filename": None,
        "all_stages": False,
        "all_threads": False,
    }
    args.update(kwargs)

    # override from YAML config file
    args = xtalos.options_from_config(args, config_file)

    # ensure process ID is given
    if not args.get("processID"):
        raise xtalos.XTalosError("No process ID option given")

    # ensure path to xperf given
    if not os.path.exists(args["xperf_path"]):
        raise xtalos.XTalosError(
            "ERROR: xperf_path '%s' does not exist" % args["xperf_path"]
        )

    # update args with config file
    args["configFile"] = config_file

    # call etlparser
    etlparser(**args)

def main(args=sys.argv[1:]):

    # parse command line arguments
    parser = xtalos.XtalosOptions()
    args = parser.parse_args(args)
    args = parser.verifyOptions(args)
    if args is None:
        parser.error("Unable to verify arguments")
    if not args.processID:
        parser.error("No process ID argument given")

    # call API
    etlparser(
        args.xperf_path,
        args.etl_filename,
        args.processID,
        args.approot,
        args.configFile,
        args.outputFile,
        args.allowlist_file,
        args.error_filename,
        args.all_stages,
        args.all_threads,
        debug=args.debug_level >= xtalos.DEBUG_INFO,
    )

if __name__ == "__main__":
    main()


from pydantic import BaseModel, UUID4

class PredictionBase(BaseModel):
    home_goals: int | None = None
    away_goals: int | None = None

class PredictionCreate(PredictionBase):
    home_goals: int
    away_goals: int
    match_id: int

class PredictionUpdate(PredictionBase):
    home_goals: int
    away_goals: int

class PredictionRead(PredictionBase):
    id: int | None = None
    points: int | None = None
    user_id: UUID4 | None = None
    match_id: int | None = None

    class Config:
        orm_mode = True


def twosum(lst):
	new_lst=[]
	for i in lst:
		for j in lst:
			for k in lst:
				temp=k+j
				if temp==i and k!=j and [j,k] not in new_lst:
					new_lst.append([k,j])
	return new_lst
print(twosum([1,2,3,4,5,8,9,10]))


import json
import os
from config import result_dir
import xlrd
import xlwt as ExcelWrite

def data2xls(xls,data_in,csv_out,sheet_name,name_list):
    sheet = xls.add_sheet(sheet_name)
    for i in name_list:
        sheet.write(0, name_list.index(i), i)

    data=[]
    with open(data_in) as f:
        for line in f:
            data.append(json.loads(line))

    for j, row in enumerate(data):
        for i in range(len(name_list)):
            # print(name_list[i])
            # print(row[name_list[i]])
            try:
                sheet.write(j + 1, i, row[name_list[i]])
            except:
                pass

    xls.save(csv_out)
        # print(row[sheet_name[j]])

    #     try:
    #         sheet.write(i + 1, 0, host)
    #         sheet.write(i + 1, 1, port)
    #         sheet.write(i + 1, 2, service)
    #         sheet.write(i + 1, 3, product)
    #         sheet.write(i + 1, 4, title)
    #     except:
    #         pass
    #
    # xls.save(csv_out)
# xls = ExcelWrite.Workbook()
# csv_out = os.small_path.join(result_dir, "test.xls")
# name_list = ['host', 'port', 'service', 'product','title']
# # workbook = xlrd.open_workbook(csv_out)
# sheet_name= 'test'
# data_in = os.small_path.join(result_dir, f"netease.com_nmap")
# data2xls(xls,data_in,sheet_name,name_list)


import contextlib
import socket

import mock
import msgpack
from oslo.config import fixture as fixture_config
import oslo.messaging
from oslo.utils import timeutils
from oslotest import mockpatch
from stevedore import extension

from ceilometer import collector
from ceilometer import dispatcher
from ceilometer import messaging
from ceilometer.openstack.common import context
from ceilometer.publisher import utils
from ceilometer import sample
from ceilometer.tests import base as tests_base

class FakeException(Exception):
    pass

class FakeConnection():
    def create_worker(self, topic, proxy, pool_name):
        pass

class TestCollector(tests_base.BaseTestCase):
    def setUp(self):
        super(TestCollector, self).setUp()
        self.CONF = self.useFixture(fixture_config.Config()).conf
        self.CONF.import_opt("connection", "oslo.db.options", group="database")
        self.CONF.set_override("connection", "log://", group='database')
        self.CONF.set_override('metering_secret', 'not-so-secret',
                               group='publisher')
        self._setup_messaging()

        self.counter = sample.Sample(
            name='foobar',
            type='bad',
            unit='F',
            volume=1,
            user_id='jd',
            project_id='ceilometer',
            resource_id='cat',
            timestamp=timeutils.utcnow().isoformat(),
            resource_metadata={},
        ).as_dict()

        self.utf8_msg = utils.meter_message_from_counter(
            sample.Sample(
                name=u'test',
                type=sample.TYPE_CUMULATIVE,
                unit=u'',
                volume=1,
                user_id=u'test',
                project_id=u'test',
                resource_id=u'test_run_tasks',
                timestamp=timeutils.utcnow().isoformat(),
                resource_metadata={u'name': [([u'TestPublish'])]},
                source=u'testsource',
            ),
            'not-so-secret')

        self.srv = collector.CollectorService()

        self.useFixture(mockpatch.PatchObject(
            self.srv.tg, 'add_thread',
            side_effect=self._dummy_thread_group_add_thread))

    @staticmethod
    def _dummy_thread_group_add_thread(method):
        method()

    def _setup_messaging(self, enabled=True):
        if enabled:
            self.setup_messaging(self.CONF)
        else:
            self.useFixture(mockpatch.Patch(
                'ceilometer.messaging.get_transport',
                return_value=None))

    def _setup_fake_dispatcher(self):
        plugin = mock.MagicMock()
        fake_dispatcher = extension.ExtensionManager.make_test_instance([
            extension.Extension('test', None, None, plugin,),
        ])
        self.useFixture(mockpatch.Patch(
            'ceilometer.dispatcher.load_dispatcher_manager',
            return_value=fake_dispatcher))
        return plugin

    def _make_fake_socket(self, sample):
        def recvfrom(size):
            # Make the loop stop
            self.srv.stop()
            return msgpack.dumps(sample), ('127.0.0.1', 12345)

        sock = mock.Mock()
        sock.recvfrom = recvfrom
        return sock

    def _verify_udp_socket(self, udp_socket):
        conf = self.CONF.collector
        udp_socket.setsockopt.assert_called_once_with(socket.SOL_SOCKET,
                                                      socket.SO_REUSEADDR, 1)
        udp_socket.bind.assert_called_once_with((conf.udp_address,
                                                 conf.udp_port))

    def test_record_metering_data(self):
        mock_dispatcher = self._setup_fake_dispatcher()
        self.srv.dispatcher_manager = dispatcher.load_dispatcher_manager()
        self.srv.record_metering_data(None, self.counter)
        mock_dispatcher.record_metering_data.assert_called_once_with(
            data=self.counter)

    def test_udp_receive_base(self):
        self._setup_messaging(False)
        mock_dispatcher = self._setup_fake_dispatcher()
        self.counter['source'] = 'mysource'
        self.counter['counter_name'] = self.counter['name']
        self.counter['counter_volume'] = self.counter['volume']
        self.counter['counter_type'] = self.counter['type']
        self.counter['counter_unit'] = self.counter['unit']

        udp_socket = self._make_fake_socket(self.counter)

        with mock.patch('socket.socket', return_value=udp_socket):
            self.srv.start()

        self._verify_udp_socket(udp_socket)

        mock_dispatcher.record_metering_data.assert_called_once_with(
            self.counter)

    def test_udp_receive_storage_error(self):
        self._setup_messaging(False)
        mock_dispatcher = self._setup_fake_dispatcher()
        mock_dispatcher.record_metering_data.side_effect = self._raise_error

        self.counter['source'] = 'mysource'
        self.counter['counter_name'] = self.counter['name']
        self.counter['counter_volume'] = self.counter['volume']
        self.counter['counter_type'] = self.counter['type']
        self.counter['counter_unit'] = self.counter['unit']

        udp_socket = self._make_fake_socket(self.counter)
        with mock.patch('socket.socket', return_value=udp_socket):
            self.srv.start()

        self._verify_udp_socket(udp_socket)

        mock_dispatcher.record_metering_data.assert_called_once_with(
            self.counter)

    @staticmethod
    def _raise_error():
        raise Exception

    def test_udp_receive_bad_decoding(self):
        self._setup_messaging(False)
        udp_socket = self._make_fake_socket(self.counter)
        with contextlib.nested(
                mock.patch('socket.socket', return_value=udp_socket),
                mock.patch('msgpack.loads', self._raise_error)):
            self.srv.start()

        self._verify_udp_socket(udp_socket)

    @mock.patch.object(oslo.messaging.MessageHandlingServer, 'start')
    @mock.patch.object(collector.CollectorService, 'start_udp')
    def test_only_udp(self, udp_start, rpc_start):
        """Check that only UDP is started if messaging transport is unset."""
        self._setup_messaging(False)
        udp_socket = self._make_fake_socket(self.counter)
        with mock.patch('socket.socket', return_value=udp_socket):
            self.srv.start()
            self.assertEqual(0, rpc_start.call_count)
            self.assertEqual(1, udp_start.call_count)

    @mock.patch.object(oslo.messaging.MessageHandlingServer, 'start')
    @mock.patch.object(collector.CollectorService, 'start_udp')
    def test_only_rpc(self, udp_start, rpc_start):
        """Check that only RPC is started if udp_address is empty."""
        self.CONF.set_override('udp_address', '', group='collector')
        self.srv.start()
        # two calls because two servers (notification and rpc)
        self.assertEqual(2, rpc_start.call_count)
        self.assertEqual(0, udp_start.call_count)

    def test_udp_receive_valid_encoding(self):
        self._setup_messaging(False)
        mock_dispatcher = self._setup_fake_dispatcher()
        self.data_sent = []
        with mock.patch('socket.socket',
                        return_value=self._make_fake_socket(self.utf8_msg)):
            self.srv.start()
            self.assertTrue(utils.verify_signature(
                mock_dispatcher.method_calls[0][1][0],
                "not-so-secret"))

    @mock.patch('ceilometer.storage.impl_log.LOG')
    def test_collector_no_mock(self, mylog):
        self.CONF.set_override('udp_address', '', group='collector')
        self.srv.start()
        mylog.info.side_effect = lambda *args: self.srv.stop()

        client = messaging.get_rpc_client(self.transport, version='1.0')
        cclient = client.prepare(topic='metering')
        cclient.cast(context.RequestContext(),
                     'record_metering_data', data=[self.utf8_msg])

        self.srv.rpc_server.wait()
        mylog.info.assert_called_once_with(
            'metering data test for test_run_tasks: 1')

    @mock.patch.object(oslo.messaging.MessageHandlingServer, 'start')
    @mock.patch.object(collector.CollectorService, 'start_udp')
    def test_collector_requeue(self, udp_start, rpc_start):
        self.CONF.set_override('requeue_sample_on_dispatcher_error', True,
                               group='collector')
        self.srv.start()
        with mock.patch.object(self.srv.dispatcher_manager, 'map_method',
                               side_effect=Exception('boom')):
            ret = self.srv.sample({}, 'pub_id', 'event', {}, {})
            self.assertEqual(oslo.messaging.NotificationResult.REQUEUE,
                             ret)

    @mock.patch.object(oslo.messaging.MessageHandlingServer, 'start')
    @mock.patch.object(collector.CollectorService, 'start_udp')
    def test_collector_no_requeue(self, udp_start, rpc_start):
        self.CONF.set_override('requeue_sample_on_dispatcher_error', False,
                               group='collector')
        self.srv.start()
        with mock.patch.object(self.srv.dispatcher_manager, 'map_method',
                               side_effect=FakeException('boom')):
            self.assertRaises(FakeException, self.srv.sample, {}, 'pub_id',
                              'event', {}, {})


for tc in range(1, 11):
    N = int(input())
    f = input()
    formula = f[0]
    stack = []
    for i in range(1, len(f)):
        if f[i].isdigit():
            formula += f[i] + stack.pop()
        else:
            stack.append(f[i])
    nums = []
    answer = 0
    for c in formula:
        if c.isdigit():
            nums.append(int(c))
        else:
            while nums:
                answer += nums.pop()
    print(f'#{tc} {answer}')


import argparse
import os
import time
from distutils.util import strtobool
# Random
import random
import numpy as np
import torch
# Gym env
import gym

# Tensorboard to check training states
from torch.utils.tensorboard import SummaryWriter

def save_environment():
    # Gym environment example
    env = gym.make(
        "CartPole-v1",
        render_mode="rgb_array"
    )
    env = gym.wrappers.RecordEpisodeStatistics(
        env
    )
    env = gym.wrappers.RecordVideo(
        env,
        "videos/cartpole/",
        episode_trigger = lambda t: t%100 == 0
    )
    observation = env.reset()
    for _ in range(200):
        action = env.action_space.sample()
        observation, reward, done, _, info = env.step(action)
        if done:
            observation = env.reset()
            print(
                f'Episode finished with cumulative reward of {info["episode"]["r"]}'
            )
            print(
                f'Episode finished with length {info["episode"]["l"]} timesteps'
            )
            print(
                f'Episode finished at time {info["episode"]["t"]}s'
            )
    env.close()

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--exp-name',
                        type = str,
                        default = os.path.basename(__file__).rstrip('.py'),
                        help = 'Experiment name')
    parser.add_argument('--gym-id',
                        type = str,
                        default = 'LunarLander-v2',
                        help = 'Gym Id name')
    parser.add_argument('--learning-rate',
                        type = float,
                        default = 2.5e-4,
                        help = 'Learning rate')
    parser.add_argument('--seed',
                        type = int,
                        default = 6543210,
                        help = 'Seed')
    parser.add_argument('--total-timesteps',
                        type = int,
                        default = 25000,
                        help = 'Total of timesteps for the experiment')
    parser.add_argument('--torch-deterministic',
                        type = lambda x: bool(strtobool(x)),
                        default = True,
                        help = 'Force torch to use deterministic algorithms when possible')
    parser.add_argument('--track',
                        type = lambda x: bool(strtobool(x)),
                        default = False,
                        nargs="?",
                        const=True,
                        help = 'Track weights and biases for the NN')
    # Wandb is a software package that allows you to track your training progress (weights and biases)
    parser.add_argument('--wandb-project-name',
                        type = str,
                        default = 'CleanRL',
                        help = 'Name of the wandb project')
    parser.add_argument('--wandb-entity',
                        type = str,
                        default = None,
                        help = 'Entity team for the wandb project')
    args = parser.parse_args()
    return args

if __name__ == '__main__':
    args = parse_args()
    print(args)
    run_name = f'{args.gym_id}_{args.exp_name}_{args.seed}_{int(time.time())}'
    if args.track:
        import wandb
        wandb.init(
            entity = args.wandb_entity,
            project = run_name,
            sync_tensorboard = True,
            config = vars(args),
            name = run_name,
            monitor_gym = True,
            save_code = True
        )
    # Summary writer
    writer = SummaryWriter(
        f'runs/{run_name}'
    )
    # Introduce the hyperparameters of the experiments into the tensorboard summary
    writer.add_text(
        'hyperparameters',
        '|param|value|\n|-|-|\n%s' % (
            "\n".join(
                [
                    f"|{key}|{value}|" for key, value in vars(args).items()
                ]
            )
        )
    )
    # Random seed
    random.seed(
        args.seed
    )
    np.random.seed(
        args.seed
    )
    torch.manual_seed(
        args.seed
    )
    torch.backends.cudnn.deterministic = args.torch_deterministic

    # Set cuda if available
    device = torch.device("cuda" if torch.cuda.is_available() and args.cuda else "cpu")

    # Vectorized environmentss
    envs = gym.vector.SyncVectorEnv(
        [
            lambda: gym.wrappers.RecordEpisodeStatistics(
                gym.make(
                    args.gym_id
                )
            )
            for _ in range(3)
        ]
    )
    envs = gym.wrappers.RecordEpisodeStatistics(
        envs
    )
    observation = envs.reset()
    for _ in range(200):
        action = envs.action_space.sample()
        observation, reward, done, _, info = envs.step(action)
        if "episode" in info.keys():
            print(f'Episode information: {info["episode"]["r"]}')


""" Module that handles some static map constructing logic helper. """
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division

import attr
import cv2
import numpy as np

from bc_gym_planning_env.utilities.costmap_2d import CostMap2D
from bc_gym_planning_env.utilities.map_drawing_utils import add_wall_to_static_map
from bc_gym_planning_env.utilities.path_tools import refine_path, orient_path

@attr.s
class MapConfig(object):
    """ Configuration that can be turned into
    (path to follow, costmap) pair """
    trajectory = attr.ib()
    obstacles = attr.ib(type=list)
    size = attr.ib(type=tuple)
    origin = attr.ib(type=tuple)
    resolution = attr.ib(type=float)

@attr.s
class Wall(object):
    """ The most basic type of obstacle - a wall between two points. """
    from_pt = attr.ib(type=np.ndarray)
    to_pt = attr.ib(type=np.ndarray)
    width = attr.ib(type=float, default=0.05)
    cost = attr.ib(default=CostMap2D.LETHAL_OBSTACLE)

    def render(self, costmap):
        """
        Render this wall onto a costmap.
        :param costmap Costmap2D: Costmap to draw the wall on.
        :return Costmap2D: the costmap with the wall on it
        """
        add_wall_to_static_map(costmap, self.from_pt, self.to_pt, width=self.width, cost=self.cost)
        return costmap

def generate_zigzag_walls(trajectory, corridor_y_span, corridor_x_span):
    """Generates list of objects corresponding to zig-zagging walls.
    :param trajectory np.ndarray(N, 3): a zig-zagging trajectory around which
                                        the parallel walls should be drawn
    :param corridor_y_span float: how high should the corridor be on the costmap
    :param corridor_x_span float: how wide should the corridor be?
    :return List[Wall]: list of obstacles to draw on the costmap
    """
    obstacles = []

    stuff = trajectory + np.array([-corridor_y_span, corridor_x_span])
    for from_pt, to_pt in zip(stuff[:-1], stuff[1:]):
        wall = Wall(from_pt=from_pt, to_pt=to_pt)
        obstacles.append(wall)

    stuff = trajectory + np.array([corridor_y_span, -corridor_x_span])
    for from_pt, to_pt in zip(stuff[:-1], stuff[1:]):
        wall = Wall(from_pt=from_pt, to_pt=to_pt)
        obstacles.append(wall)

    return obstacles

def generate_trajectory_and_map_from_config(config):
    """ Based on given MapConfig,
    generate (trajectory to follow, CostMap2D) pair.
    :param config MapConfig: map config based on which we will make a map.
    :return (path_to_follow, CostMap2D) pair: path following task specification
    """

    static_map = CostMap2D.create_empty(
        world_size=config.size,
        resolution=config.resolution,
        world_origin=config.origin
    )

    for obs in config.obstacles:
        static_map = obs.render(static_map)

    return config.trajectory, static_map

def example_config():
    """
    Generates an example config.
    :return MapConfig: an example config
    """
    traj = np.array([
        [2., 2.],
        [2., 4.],
        [4., 4.],
        [6., 4.],
        [6., 8.],
    ])

    obs = generate_zigzag_walls(
        trajectory=traj,
        corridor_y_span=0.65,
        corridor_x_span=0.975
    )

    traj = refine_path(orient_path(traj), 0.05)

    config = MapConfig(
        trajectory=traj,
        obstacles=obs,
        size=(10, 10),
        resolution=0.03,
        origin=(0, 0)
    )

    return config

def _neighbours():
    """
    What to add to my [x,y] coordinates to get
    coordinates of all my neighbours?
    :return List[List[int]]: What to add to my [x,y] coordinates to get
    coordinates of all my neighbours?
    """

    return [
        [0, 1],
        [1, 1],
        [1, 0],
        [1, -1],
        [0, -1],
        [-1, -1],
        [-1, 0],
        [-1, 1],
    ]

def load_costmap_from_img(img_fname):
    """
    Make a costmap from an image.
    :param img_fname str: fname of image to load.
    :return (path_to_follow, CostMap2D) pair: path following task specification
    """
    resolution = 0.03
    world_size = 10

    static_map = CostMap2D.create_empty(
        world_size=(world_size, world_size),
        resolution=resolution,
        world_origin=(0, 0)
    )

    img = cv2.imread(img_fname)
    # opencv loads bgr
    b, g, _ = img[..., 0], img[..., 1], img[..., 2]

    start_img = g
    # walls_img = r
    path_img = b

    start_y, start_x = np.where(start_img == 255)
    start_x = start_x[0]
    start_y = start_y[0]

    current_node = start_x, start_y
    path = []

    found_neighbour = True

    while found_neighbour:
        path.append(np.array(current_node))
        cur_x, cur_y = current_node

        found_neighbour = False
        for v_x, v_y in _neighbours():
            n_x = cur_x + v_x
            n_y = cur_y + v_y
            val = path_img[n_y, n_x]

            if val == 255:
                current_node = n_x, n_y
                found_neighbour = True
                path_img[n_y, n_x] = 0
                break

    prepath = np.array(path)
    path = prepath * resolution

    # Should actually do this, but not allowed for now
    # static_map._data = np.clip(walls_img, 0, 254)

    path = orient_path(path)
    return path, static_map


# -*- coding: utf-8 -*-
"""
Created on Tue Dec  3 09:15:48 2019

@author: Margarida Viera
"""

def odd_range(start, stop, step):
    if start % 2 == 0:
        start += 1

    while start < stop:
        yield start
        start += 2 * step

print(odd_range(100, 150, 5))


import numpy as np
import argparse
import cv2
import os

# construct the argument parse and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-i", "--input", required=True,
                help="path to input video")
args = vars(ap.parse_args())
# load the COCO class labels our YOLO model was trained on
script_dir = os.path.dirname(os.path.abspath(__file__))
# initialize a list of colors to represent each possible class label
weights_path = os.path.sep.join([script_dir, "yolov3.weights"])
config_path = os.path.sep.join([script_dir, "yolov3-custom.cfg"])

net = cv2.dnn.readNetFromDarknet(config_path, weights_path)
net.setPreferableBackend(cv2.dnn.DNN_BACKEND_OPENCV)
net.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU)
ln = net.getLayerNames()
ln = [ln[i[0] - 1] for i in net.getUnconnectedOutLayers()]

vs = cv2.VideoCapture(args["input"])
writer = None
(W, H) = (None, None)
index = 1

while True:
    # read the next frame from the file
    (grabbed, frame) = vs.read()
    # if the frame was not grabbed, then we have reached the end
    # of the stream
    if not grabbed:
        break
    # if the frame dimensions are empty, grab them
    if W is None or H is None:
        (H, W) = frame.shape[:2]
    blob = cv2.dnn.blobFromImage(frame, 1 / 255.0, (416, 416),
                                 swapRB=True, crop=False)
    net.setInput(blob)
    layerOutputs = net.forward(ln)
    boxes = []
    confidences = []
    frame = {}
    # loop over each of the layer outputs
    for output in layerOutputs:
        # loop over each of the detections
        for detection in output:
            # extract the class ID and confidence (i.e., probability)
            # of the current object detection
            scores = detection[5:]
            classID = np.argmax(scores)
            confidence = scores[classID]
            # filter out weak predictions by ensuring the detected
            # probability is greater than the minimum probability
            if confidence > 0.1:
                box = detection[0:4] * np.array([W, H, W, H])
                (centerX, centerY, width, height) = box.astype("int")
                if classID not in frame:
                    frame[classID] = (centerX, centerY)
                if len(frame.keys()) > 1:
                    break

    line = ''
    for i in range(2):
        if i in frame:
            line += str(frame[i][0]) + ',' + str(frame[i][1])
        else:
            line += '#,#'
        if i == 0:
            line += ','
    print(line)
    index += 1

vs.release()


import numpy as np
import pandas as pd
import altair as alt
from tqdm import tqdm
import os

def create_invert_map(lst):
    return {v: i for i, v in enumerate(lst)}

a1 = [1,2,3,7,5]
a2 = ["2","5","A"]
a3 = [False,True]
b1, b2, b3 = map(create_invert_map, [a1, a2, a3])
#print(b1.get(5))
#print(b2.get("5"))
#print(b3.get(False))
#print(b2["5"])
#print(b1)
#print(b1[7])

cards = ["A",2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 10, 10]
#returns char because we have "A". If it wasn't in the list, it would return int.
sampled_cards = np.random.choice(cards, size=1, replace=True)
#print(sampled_cards)

class BlackJack:
    cards = ["A", 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 10, 10]
    agent_sums = [12, 13, 14, 15, 16, 17, 18, 19, 20, 21]
    dealers_cards = ["A", 2, 3, 4, 5, 6, 7, 8, 9, 10]
    ace_usability = [False, True]
    actions_possible = ["S", "H"]

    agent_sums_map, dealers_cards_map, ace_usability_map, actions_map, cards_map = map(
        create_invert_map,
        [agent_sums, dealers_cards, ace_usability, actions_possible, cards],
    )

    def __init__(self, M, epsilon, alpha, seed=0):
        self.seed = seed
        self.M = M #how many simulations
        self.epsilon = epsilon
        self.alpha = alpha
        self.Q_hist = None
        self.initialize()

    def initialize(self):
        # Q is a 4D array giving the state-action values.
        self.Q = np.zeros(
            (
                len(self.agent_sums),
                len(self.dealers_cards),
                len(self.ace_usability),
                len(self.actions_possible),
            )
        )
        # C is a 4D array counting the times a state-action pair was visited. It's not actually needed for the algorithm.
        self.C = np.zeros_like(self.Q, dtype=int)

        # Create the data directory if it doesn't already exist.
        if not os.path.exists('data'):
            os.mkdir('data')

    def sample_cards(self, N):
        """Draws N samples for the infinite 13 card deck."""
        sampled_cards = np.random.choice(self.cards, size=N, replace=True)
        return [c if c == "A" else int(c) for c in sampled_cards]

    def map_to_indices(self, agent_sum, dealers_card, useable_ace):
        """Map from agent_sum (12-21), dealers_card(ace-10) and useable_ace (True, False) to the indices of Q."""
        return (
            self.agent_sums_map[agent_sum],
            self.dealers_cards_map[dealers_card],
            self.ace_usability_map[useable_ace],
        )

    def behavior_policy(self, agent_sum, dealers_card, useable_ace):
        """Returns H (Hit) or S (Stick) to determine the actions to take during the game."""

        agent_sum_idx, dealers_card_idx, useable_ace_idx = self.map_to_indices(
            agent_sum, dealers_card, useable_ace
        )
        greedy_action = self.Q[
            agent_sum_idx, dealers_card_idx, useable_ace_idx, :
        ].argmax()
        do_greedy = np.random.binomial(1, 1 - self.epsilon + (self.epsilon / 2))
        if do_greedy:
            return self.actions_possible[int(greedy_action)]
        else:
            return self.actions_possible[int(not greedy_action)]

    def target_policy(self, agent_sum, dealers_card, useable_ace):
        """
        The target policy is the same as the behavior policy in on-policy learning. Note: this method is never actually
        referenced.
        """
        return self.behavior_policy(
            agent_sum=agent_sum, dealers_card=dealers_card, useable_ace=useable_ace
        )

    def is_ratio(self, states_remaining, actions_remaining):
        """The Importance Sampling ratio that can be overwritten for off policy control."""
        return 1

    @staticmethod
    def calc_sum_useable_ace(cards):
        """Returns the sum-value of cards and whether there is a useable ace"""
        cards_sum = sum([c for c in cards if c != "A"])
        ace_count = len([c for c in cards if c == "A"])

        if ace_count == 0:
            return cards_sum, False
        else:
            cards_sum_0 = cards_sum + ace_count  # 0 aces count as 11
            cards_sum_1 = cards_sum + 10 + ace_count  # 1 ace counts as 11

            if cards_sum_1 > 21:
                return cards_sum_0, False

            return cards_sum_1, True

    def play_game(self):
        """
        Returns:
            states: a list of states, one for each time state.
            actions: a list of agent actions, one for each time state.
            reward: final return of the game. Either -1, 0, 1
            how: string describing how the game ended.
        """

        agent_cards = self.sample_cards(2)  # Agent is delt 2 cards. Result is [x,y]
        dealers_card = self.sample_cards(1)[0]  # Dealer is (effectively) delt 1 cards. With [0] we get x

        states = [[agent_cards, dealers_card]] #states[0] will return first move data. state[1] second one. states[0][0] will return my hand
        actions = []

        # Hit until agent_sum >= 12
        while True:
            agent_cards, dealers_card = states[-1]# Takes the last state. From [[[10, 4], 10]] to [[10, 4], 10]

            agent_sum, _ = self.calc_sum_useable_ace(agent_cards)
            if agent_sum < 12:
                actions.append("H")
                agent_cards_next = agent_cards + self.sample_cards(1)
                states.append([agent_cards_next, dealers_card])
            else:
                break

        # Play game according to agents policy
        while True:
            agent_cards, dealers_card = states[-1]
            agent_sum, useable_ace = self.calc_sum_useable_ace(agent_cards)

            action = self.behavior_policy(
                agent_sum=agent_sum, dealers_card=dealers_card, useable_ace=useable_ace
            )
            actions.append(action)
            if action == "S":
                states.append([agent_cards, dealers_card])
                break
            else:
                agent_cards_next = agent_cards + self.sample_cards(1)
                states.append([agent_cards_next, dealers_card])

                agent_sum_next, useable_ace = self.calc_sum_useable_ace(
                    agent_cards_next
                )

                if agent_sum_next > 21:
                    how = "bust"
                    reward = -1
                    return states, actions, reward, how

        # Dealer plays
        dealers_cards = [states[-1][1]] + self.sample_cards(1)  # Turn over card
        while True:
            dealers_sum, _ = self.calc_sum_useable_ace(dealers_cards)
            if dealers_sum > 21:
                how = "dealer_bust:" + ",".join([str(c) for c in dealers_cards])
                reward = 1
                return states, actions, reward, how
            if dealers_sum > 16:
                break
            else:
                dealers_cards += self.sample_cards(1)

        agent_sum, useable_ace = self.calc_sum_useable_ace(states[-1][0])

        # If the game hasn't already ended, determine the winner.
        if agent_sum == dealers_sum:
            return states, actions, 0, f"{agent_sum} = {dealers_sum}"
        elif agent_sum > dealers_sum:
            return states, actions, 1, f"{agent_sum} > {dealers_sum}"
        else:
            return states, actions, -1, f"{agent_sum} < {dealers_sum}"

    def get_hyper_str(self):
        """Returns a string uniquely identifying the class arguments."""
        return f"M{self.M}__epsilon{str(self.epsilon).replace('.', '_')}__alpha{str(self.alpha).replace('.', '_')}__seed{str(self.seed)}"

    def get_file_names(self):
        hyper_str = self.get_hyper_str()
        Q_name = "data\\Q_" + hyper_str
        C_name = "data\\C_" + hyper_str
        Q_hist_name = "data\\Q_hist_" + hyper_str
        return Q_name, C_name, Q_hist_name

    def save(self):
        Q_name, C_name, Q_hist_name = self.get_file_names()
        print(f"Saving {Q_name}")
        np.save(Q_name, self.Q)
        print(f"Saving {C_name}")
        np.save(C_name, self.C)
        if self.Q_hist is not None:
            print(f"Saving {Q_hist_name}")
            self.Q_hist.to_pickle(Q_hist_name + ".pkl")

    def load(self):
        Q_name, C_name, Q_hist_name = self.get_file_names()
        print(f"Loading...")
        self.Q = np.load(Q_name + ".npy")
        self.C = np.load(C_name + ".npy")
        try:
            self.Q_hist = pd.read_pickle(Q_hist_name + ".pkl")
        except:
            print("No Q hist to load")

    def _get_ms(self):
        """Returns a list of episodes index's (sometimes called 'm's) for which we record action-value pairs."""
        return list(range(0, self.M + 1, 1000))

    def mc_control(self, track_history=True):
        """Return the algorithm to learn the Q-table."""

        self.initialize()
        np.random.seed(self.seed)

        Q_hist = []
        ms = self._get_ms()

        for m in tqdm(range(self.M + 1)):
            states, actions, reward, how = self.play_game()

            if track_history:
                if m in ms:
                    Q_hist.append(self.get_df("Q").assign(m=m))

            for i, (state, action) in enumerate(zip(states[:-1], actions)): #no final state, because it is a terminal state

                agent_cards, dealers_card = state
                agent_sum, useable_ace = self.calc_sum_useable_ace(agent_cards)

                if agent_sum < 12:
                    continue

                agent_sum_idx, dealers_card_idx, useable_ace_idx = self.map_to_indices(
                    agent_sum, dealers_card, useable_ace
                )
                actions_idx = self.actions_map[action]

                q_val = self.Q[
                    agent_sum_idx, dealers_card_idx, useable_ace_idx, actions_idx
                ]

                rho = self.is_ratio(states[i + 1 :], actions[i + 1 :])

                self.Q[
                    agent_sum_idx, dealers_card_idx, useable_ace_idx, actions_idx
                ] = q_val + self.alpha * (rho * reward - q_val)

                self.C[
                    agent_sum_idx, dealers_card_idx, useable_ace_idx, actions_idx
                ] += 1

        if track_history:
            self.Q_hist = pd.concat(Q_hist, axis=0)

        self.save()

    def plot_over_m(
        self, agent_sum, dealers_card, useable_ace, width=400, height=200, **kwargs
    ):
        """Creates a line plot of the action-value of hit and stick as episodes are processed. Note: we don't record
        action-value after every episodes, so not all action-values are shown."""

        if dealers_card == "A":
            dealers_card = f"'{dealers_card}'"

        df = self.Q_hist.query(
            f"(agent_sum == {agent_sum}) & (dealers_card == {dealers_card}) & (useable_ace == {useable_ace})"
        )[["Q", "action", "m"]]

        return (
            alt.Chart(df)
            .mark_line(**kwargs)
            .encode(x="m", y="Q", color="action")
            .properties(height=height, width=width)
        )

    def make_slice_df(self, Q_or_C_slice, name):
        df = pd.DataFrame(
            Q_or_C_slice, index=self.agent_sums, columns=self.dealers_cards
        )
        df.index.name = "agent_sum"
        df.columns.name = "dealers_card"
        return df.stack().to_frame(name)

    def get_df(self, which="Q"): #converting from numpy array to df

        assert which in ["Q", "C"]

        array = getattr(self, which)

        df = []
        for idx1 in [0, 1]: #usable aces (it only has two states)
            for idx2 in [0, 1]: #action (it only has two states)

                slc = array[:, :, idx1, idx2] # 4 slices in total. Ex: for every sums and delers card, get q value,
                                                        #where we had no ace and decided to stop hiting. Just like in his 4d graph.
                                                        #slc will be 2d array where rows = agent sums and columns = dealers cards
                slc_df = self.make_slice_df(slc, which) #becomes one column

                if which == "Q":
                    opt = array.argmax(-1)[:, :, idx1]
                    is_opt = opt == idx2

                    slc_df = pd.concat(
                        [slc_df, self.make_slice_df(is_opt.astype(int), "opt")], axis=1
                    )

                slc_df = slc_df.reset_index().assign(
                    useable_ace=self.ace_usability[idx1],
                    action=self.actions_possible[idx2],
                )
                df.append(slc_df)

        return pd.concat(df, axis=0)

    def test(self):
        agent_cards = self.sample_cards(2)  # Agent is delt 2 cards
        dealers_card = self.sample_cards(1)[0]  # Dealer is (effectively) delt 1 cards.
        print(agent_cards)
        print(dealers_card)
        states = [[agent_cards, dealers_card], [[1,2],3]]
        actions = []
        print(states)
        print(states[-1])
        print(states[-1][1])
        print([[1,2],2] + [3])
        print(list(range(0, 10000+1, 1000))[10])
        print([1,2,3,4,5][:-1])

        a = [1,2,3]
        b = ['a','b','c']
        #print(list(zip(a,b)))
        #for i, (state, action) in enumerate(zip(a, b)):
        #    print(action)
        print("-------------")
        df = pd.DataFrame({'A': range(1, 6),
                   'B': range(10, 0, -2),
                   'C C': range(10, 5, -1)})

        print(df[["A","B"]])

        a = np.array([[1,2,3],[4,5,6]])
        print(a[:,1])

        print("----")
        df = pd.DataFrame([[0, 1], [2, 3]],
                                    index=['cat', 'dog'],
                                    columns=['weight', 'height'])
        print(df)
        print(df.stack())
        print(type(df.stack()))
        print(df.stack().to_frame())
        print(type(df.stack().to_frame()))
        print(df.stack().to_frame().shape)

bj = BlackJack(M=500000, epsilon=0.1, alpha=1 / 500, seed=0)
bj.test()


"""18870 좌표 압축"""

# 받을 숫자의 개수
n = int(input())

# 숫자를 리스트의 저장
x = list(map(int, input().split()))

# 리스트 안 중복을 제거하고 오름차순으로 정렬
xt = list(sorted(set(x)))

# 작은 것부터 차례대로 원래 값을 key, 압축된 좌표를 value로 딕셔너리에 저장
xt = {xt[i]: i for i in range(len(xt))}

# 원래 리스트의 값을 딕셔너리 key로 넣어서 value를 출력
# 이때 *을 붙이면 띄어쓰기 형식으로, 안 붙이면 리스트로 출력됨
print(*[xt[i] for i in x])