import logging
import os.path
import re
from argparse import ArgumentParser
from collections import defaultdict
from dataclasses import dataclass
from typing import Any, Dict, Iterator, List, NamedTuple, Optional, Union, cast

from common import directories, file_utils
from common.commands.base import ArxivBatchCommand
from common.parse_tex import PhraseExtractor, overlaps
from common.types import ArxivId, CharacterRange, FileContents
from tqdm import tqdm
from typing_extensions import Literal

from ..nlp import DefinitionDetectionModel
from ..types import Definiendum, Definition, EmbellishedSentence, TermReference

"""
Deployment TODOs:
* TODO(dykang): document model download instructions in README
* TODO(dykang): make model location configurable in config.ini
"""

TermName = str
DefinitionId = str

@dataclass(frozen=True)
class DetectDefinitionsTask:
    arxiv_id: ArxivId
    sentences: List[EmbellishedSentence]
    tex_by_file: Dict[str, FileContents]

@dataclass(frozen=True)
class TermDefinitionPair:
    term_start: int
    term_end: int
    term_text: str
    term_type: str
    term_confidence: Optional[float]
    definition_start: int
    definition_end: int
    definition_text: str
    definition_type: str
    definition_confidence: Optional[float]

def get_token_character_ranges(text: str, tokens: List[str]) -> List[CharacterRange]:
    """
    Extract start and end charcter positions for each token in featurized tokens
    """
    ranges = []
    current_position = 0
    for token in tokens:
        start_index = text[current_position:].index(token)
        ranges.append(
            CharacterRange(
                current_position + start_index,
                current_position + start_index + len(token) - 1,
            )
        )
        current_position += len(token) + start_index
    return ranges

StringOffset = int

def get_symbol_texs(
    sentence_with_symbol_tags: str, sentence_with_formula_contents: str
) -> Optional[Dict[StringOffset, str]]:
    """
    Get a map from the character offsets of the word 'SYMBOL' in the first input string
    to the TeX for that symbol, found from the second input string.  Example:

    Input: get_symbol_texs("Add SYMBOL to SYMBOL.", "Add (((FORMULA:x))) to (((FORMULA:y)))")
    Output: { 4: "x", 14: "y" }

    Returns None if there was an error establishing a mapping, for instance if there are
    a different number of 'SYMBOL' tags between the two versions of the sentence.
    """
    symbol_starts = [
        match.start() for match in re.finditer(r"SYMBOL", sentence_with_symbol_tags)
    ]
    symbol_texs = [
        match.group(1)
        for match in re.finditer(
            # The extra '[)]*]' before the ending triple-parentheses ensures that
            # closing parentheses that were part of the original equation (e.g., for
            # a symbol like 'f(x)') are included in the symbol, rather than being read as part
            # of the triple-parens formula delimiter.
            r"\(\(\(FORMULA:(.*?[)]*)\)\)\)",
            sentence_with_formula_contents,
        )
    ]

    if len(symbol_starts) != len(symbol_texs):
        logging.warning(  # pylint: disable=logging-not-lazy
            "The two representations of a sentence %s and %s were detected as having differing "
            + "numbers of symbols. A lookup table for symbol TeX cannot be built. The TeX for "
            + "symbols in these equations may not be correct in the output.",
            sentence_with_symbol_tags,
            sentence_with_formula_contents,
        )
        return None

    return dict(zip(symbol_starts, symbol_texs))

def consolidate_keyword_definitions(
    text: str,
    tokens: List[str],
    slot_predictions: List[str],
    slot_prediction_confidences: List[float],
    prediction_type: str,
) -> List[TermDefinitionPair]:

    # Make index from tokens to their character positions.
    ranges = get_token_character_ranges(text, tokens)

    # Extract ranges for all terms and definitions.
    terms: List[List[CharacterRange]] = []
    definitions: List[List[CharacterRange]] = []
    term_confidence_list: List[float] = []
    definition_confidence_list: List[float] = []
    term_ranges, definition_ranges = [], []
    term_confidences, definition_confidences = [], []

    for (slot_label, slot_confidence, range_) in zip(
        slot_predictions, slot_prediction_confidences, ranges
    ):
        if slot_label == "TERM":
            term_ranges.append(range_)
            term_confidences.append(slot_confidence)
        if slot_label == "DEF":
            definition_ranges.append(range_)
            definition_confidences.append(slot_confidence)
        if slot_label == "O":
            if len(term_ranges) > 0:
                terms.append(term_ranges)
                term_confidence_list.append(
                    sum(term_confidences) / len(term_confidences)
                )
            if len(definition_ranges) > 0:
                definitions.append(definition_ranges)
                definition_confidence_list.append(
                    sum(definition_confidences) / len(definition_confidences)
                )
            term_ranges, definition_ranges = [], []

    if len(term_ranges) > 0:
        terms.append(term_ranges)
        term_confidence_list.append(sum(term_confidences) / len(term_confidences))
    if len(definition_ranges) > 0:
        definitions.append(definition_ranges)
        definition_confidence_list.append(
            sum(definition_confidences) / len(definition_confidences)
        )
    term_ranges, definition_ranges = [], []

    # Match pairs of term and definitions sequentially ( O T O O D then (T, D)). This
    # does not handle multi-term / definitions pairs.
    num_term_definition_pairs = min(len(terms), len(definitions))
    pairs: List[TermDefinitionPair] = []
    for pair_index in range(num_term_definition_pairs):
        logging.debug(
            "Found slot predictions for tokens: (tokens: %s), (slots: %s)",
            tokens,
            slot_predictions,
        )
        logging.debug(
            "Processing term-definition pair %d of %d",
            pair_index,
            num_term_definition_pairs,
        )

        term_rangelist = terms[pair_index]
        definition_rangelist = definitions[pair_index]
        term_confidence = term_confidence_list[pair_index]
        definition_confidence = definition_confidence_list[pair_index]

        term_start = min([term_range.start for term_range in term_rangelist])
        term_end = max([term_range.end for term_range in term_rangelist]) + 1
        definition_start = min(
            [definition_range.start for definition_range in definition_rangelist]
        )
        definition_end = (
            max([definition_range.end for definition_range in definition_rangelist]) + 1
        )

        all_definition_types = {
            "W00": ["term", "definition"],
            "AI2020": ["abbreviation", "expansion"],
            "DocDef2": ["symbol", "nickname"],
        }

        term_type = all_definition_types[prediction_type][0]
        definition_type = all_definition_types[prediction_type][1]

        pair = TermDefinitionPair(
            term_start=term_start,
            term_end=term_end,
            term_text=text[term_start:term_end],
            term_type=term_type,
            term_confidence=term_confidence,
            definition_start=definition_start,
            definition_end=definition_end,
            definition_text=text[definition_start:definition_end],
            definition_type=definition_type,
            definition_confidence=definition_confidence,
        )
        logging.debug("Found definition-term pair %s", pair)
        pairs.append(pair)

    return pairs

def check_text_contains_abbreviation_for_sanity(
    abbreviation_text: str,
    expansion_text: str,
    abbreviation_start: int,
    abbreviation_end: int,
    expansion_start: int,
    expansion_end: int,
) -> bool:
    """
    Very smooth filter:
    Outputs from our current abbreviation detector include lots of noise
        e.g., "alpha x (Ref )" or any phrases with parenthesis
    This function filters out the noise using simple heuristics.
    """
    # If abbreviation and expansion overlaps in positions, ignore them.
    if (
        len(
            set(range(abbreviation_start, abbreviation_end - 1)).intersection(
                range(expansion_start, expansion_end - 1)
            )
        )
        > 0
    ):
        return False

    # If citation patterns are detected (e.g., Citation (CITATION)), ignore them.
    if "citation" in abbreviation_text.lower():
        return False

    # Check each letter in abbreviation appears in expansion text.
    is_abbreviation = True
    for letter_in_abbreviation in abbreviation_text.lower():
        if letter_in_abbreviation not in expansion_text.lower():
            is_abbreviation = False

    return is_abbreviation

def get_abbreviations(
    text: str, tokens: List[str], nlp_model: Any
) -> List[TermDefinitionPair]:
    """
    Legacy function for detecting abbreviations in a text. No longer in use.
    """

    pairs: List[TermDefinitionPair] = []

    doc = nlp_model(text)

    # Make index from tokens to their character positions.
    token_ranges = get_token_character_ranges(text, tokens)

    for abbreviation in doc._.abbreviations:
        # Abbreviation (term).
        abbreviation_ranges = token_ranges[abbreviation.start : abbreviation.end]
        abbreviation_start = min([r.start for r in abbreviation_ranges])
        abbreviation_end = max([r.end for r in abbreviation_ranges]) + 1

        # Expansion (definition).
        expansion_ranges = token_ranges[
            abbreviation._.long_form.start : abbreviation._.long_form.end
        ]
        expansion_start = min(
            [expansion_range.start for expansion_range in expansion_ranges]
        )
        expansion_end = (
            max([expansion_range.end for expansion_range in expansion_ranges]) + 1
        )

        if not check_text_contains_abbreviation_for_sanity(
            str(abbreviation),
            str(abbreviation._.long_form),
            abbreviation_start,
            abbreviation_end,
            expansion_start,
            expansion_end,
        ):
            continue

        pair = TermDefinitionPair(
            term_start=abbreviation_start,
            term_end=abbreviation_end,
            term_text=text[abbreviation_start:abbreviation_end],
            term_type="abbreviation",
            term_confidence=None,
            definition_start=expansion_start,
            definition_end=expansion_end,
            definition_text=text[expansion_start:expansion_end],
            definition_type="expansion",
            definition_confidence=None,
        )
        logging.debug("Found abbreviation-expansion pair %s", pair)
        pairs.append(pair)

    return pairs

class Nickname(NamedTuple):
    token_indexes: List[int]
    token_ranges: List[CharacterRange]

Direction = Literal["LEFT", "RIGHT"]

def search_symbol_nickname(
    token_index: int,
    pos: List[str],
    ranges: List[CharacterRange],
    direction: Direction,
) -> Optional[Nickname]:
    """
    This function searches for a nickname pattern for a token, in the direction specified.
    Get all tokens to the left or right of the token that match the allowed POS tags.
    Return symbol and nickname indices and ranges if there are valid POS spans. Else, return None for indices and spans.
    """

    # Union set of POS tags in nicknames. Feel free to add more if you have new patterns.
    # POS Tag Expansions : https://www.ling.upenn.edu/courses/Fall_2003/ling001/penn_treebank_pos.html.
    UNION_POS_LIST = ["DT", "JJ", "NN", "NNS", "NNP", "NNPS", ":", "-LRB-"]
    pos_indexes = []
    pos_tags = []
    pos_ranges = []
    if direction == "RIGHT":
        current_index = token_index + 1
    else:
        current_index = token_index - 1

    # Exit if the current index is greater than the length of the sentence or below zero.
    if current_index >= len(pos) or current_index < 0:
        return None

    # Search ahead until the POS tag is not in the union set.
    while pos[current_index] in UNION_POS_LIST:
        pos_indexes.append(current_index)
        pos_ranges.append(ranges[current_index])
        pos_tags.append(pos[current_index])
        if direction == "RIGHT":
            current_index += 1
            if current_index >= len(pos):
                break
        else:
            current_index -= 1
            if current_index < 0:
                break

    # Ignore when the POS tag is among ["DT", ":", "-LRB-"].
    if len(pos_tags) == 1 and (pos_tags[0] in ["DT", ":", "-LRB-"]):
        return None

    if len(pos_indexes) > 0:
        if direction == "LEFT":
            nickname_indexes = list(reversed(pos_indexes))
            nickname_ranges = list(reversed(pos_ranges))
            nickname_tags = list(reversed(pos_tags))
        else:
            nickname_indexes = pos_indexes
            nickname_ranges = list(pos_ranges)
            nickname_tags = list(pos_tags)

        # Skip ["DT", ":", "-LRB-"] as first or last nickname tokens.
        if nickname_tags[0] in ["DT", ":", "-LRB-"]:
            nickname_ranges = nickname_ranges[1:]
            nickname_indexes = nickname_indexes[1:]
        elif nickname_tags[-1] in ["DT", ":", "-LRB-"]:
            nickname_ranges = nickname_ranges[:-1]
            nickname_indexes = nickname_indexes[:-1]

        return Nickname(nickname_indexes, nickname_ranges)
    else:
        return None

class SymbolNickname(NamedTuple):
    symbol: str
    symbol_index: int
    symbol_range: CharacterRange
    nickname_indexes: List[int]
    nickname_ranges: List[CharacterRange]

def get_symbol_nickname_pairs(
    text: str, tokens: List[str], pos: List[str], symbol_texs: Dict[StringOffset, str]
) -> List[TermDefinitionPair]:
    """
    Legacy function for detecting nicknames of a symbol in a text. No longer in use.
    """

    # Check whether a symbol's definition is a nickname of the symbol or not
    # using heuristic rules below, although they are not perfect for some cases.
    # a/DT particular/JJ transcript/NN SYMBOL/NN.
    # the/DT self-interaction/JJ term/JJ SYMBOL/NN.
    # the/DT main/JJ effect/NN of/NN feature/JJ SYMBOL/JJ.

    # If a new algorithm needs to be added, a separate function can be called in the iteration below. Finally, append a pair object to the 'pairs' list.
    ranges = get_token_character_ranges(text, tokens)
    pairs = []
    if "SYMBOL" in text:

        symbol_nickname_pairs: List[SymbolNickname] = []
        symbol_index = 0

        for token_index, (token, range_) in enumerate(zip(tokens, ranges)):

            # 1. If of the form '*th', check RIGHT of symbol.
            # 2. If token is a symbol, then:
            #   a. If the symbol tex is present:
            #       i. If single length symbol, first check LEFT then RIGHT.
            #       ii. If multi length symbol, check LEFT.
            #   b. If symbol tex is not present, just check LEFT.
            if token == "SYMBOLth":
                nickname = search_symbol_nickname(token_index, pos, ranges, "RIGHT")
                if nickname is not None and range_.start in symbol_texs:
                    symbol_nickname_pairs.append(
                        SymbolNickname(
                            symbol_texs[range_.start],
                            token_index,
                            range_,
                            nickname.token_indexes,
                            nickname.token_ranges,
                        )
                    )

            elif token == "SYMBOL":
                # Decide the order of LEFT or RIGHT.
                directions: List[Direction] = []
                if token_index > 0:
                    if token_index < len(pos) - 1:
                        # Search RIGHT first if the immediate next token is among ["NN", ":"].
                        if pos[token_index + 1] in ["NN", ":"]:
                            directions = ["RIGHT", "LEFT"]
                        else:
                            directions = ["LEFT", "RIGHT"]
                    else:
                        directions = ["LEFT"]
                elif token_index < len(pos) - 1:
                    directions = ["RIGHT"]

                for direction in directions:
                    nickname = search_symbol_nickname(
                        token_index, pos, ranges, direction
                    )
                    if nickname is not None and range_.start in symbol_texs:
                        symbol_nickname_pairs.append(
                            SymbolNickname(
                                symbol_texs[range_.start],
                                token_index,
                                range_,
                                nickname.token_indexes,
                                nickname.token_ranges,
                            )
                        )
                        break

            if "SYMBOL" in token:
                symbol_index += 1

        for sn in symbol_nickname_pairs:
            symbol_start = sn.symbol_range.start
            symbol_end = sn.symbol_range.end + 1
            nickname_start = min(
                [nickname_range.start for nickname_range in sn.nickname_ranges]
            )
            nickname_end = (
                max([nickname_range.end for nickname_range in sn.nickname_ranges]) + 1
            )
            nickname_text = text[nickname_start:nickname_end]
            # Reject a nickname if the nickname text is "SYMBOL".
            if nickname_text == "SYMBOL":
                continue

            pair = TermDefinitionPair(
                term_start=symbol_start,
                term_end=symbol_end,
                term_text=sn.symbol,
                term_type="symbol",
                term_confidence=None,
                definition_start=nickname_start,
                definition_end=nickname_end,
                definition_text=nickname_text,
                definition_type="nickname",
                definition_confidence=None,
            )
            logging.debug("Found symbol-nickname pair %s", pair)
            pairs.append(pair)

    return pairs

class DetectDefinitions(
    ArxivBatchCommand[
        DetectDefinitionsTask, Union[Definiendum, Definition, TermReference]
    ]
):
    """
    Extract terms and definitions of those terms from sentences using a pre-trained definition
    extraction model. Performs the following steps:
    1. Load cleaned sentences from the 'detected-sentences' data directory
    2. Extract features from each sentence (aka featurization)
    3. Load the pre-trained NLP model, load the features, and predict intent and term:definition
       slots. The model predicts intents and slots:
        * Intent: whether a sentence is detected as including a definition or not
        * Slot: a tag for each token indicating it is one of (TERM, DEFINITION, neither)
    4. Find all references to terms in the text
    5. Save detected terms to 'entities-terms.csv' and detected definitions to
       'entities-definitions.csv'
    """

    @staticmethod
    def get_name() -> str:
        return "detect-definitions"

    @staticmethod
    def get_description() -> str:
        return "Extract term and definition pairs from LaTeX."

    @staticmethod
    def init_parser(parser: ArgumentParser) -> None:
        super(DetectDefinitions, DetectDefinitions).init_parser(parser)
        parser.add_argument(
            "--show-progress",
            action="store_true",
            help=(
                "Whether to show progress bar during definition extraction. "
                + "Interferes with log messages shown on the command line."
            ),
        )
        parser.add_argument(
            "--batch-size",
            type=int,
            default=8,
            help=("Number of sentences to process at a time to detect definitions."),
        )

    def get_arxiv_ids_dirkey(self) -> str:
        return "detected-sentences"

    def load(self) -> Iterator[DetectDefinitionsTask]:
        for arxiv_id in self.arxiv_ids:

            output_dir = directories.arxiv_subdir("detected-definitions", arxiv_id)
            file_utils.clean_directory(output_dir)

            # Load cleaned sentences for definition detection.
            detected_sentences_path = os.path.join(
                directories.arxiv_subdir("sentence-tokens", arxiv_id), "sentences.csv",
            )
            try:
                sentences = list(
                    file_utils.load_from_csv(
                        detected_sentences_path, EmbellishedSentence
                    )
                )
            except FileNotFoundError:
                logging.warning(  # pylint: disable=logging-not-lazy
                    "No sentences data found for arXiv paper %s. Try re-running the pipeline, "
                    + "this time enabling the processing of sentences. If that doesn't work, "
                    + "there is likely an error in detected sentences for this paper.",
                    arxiv_id,
                )
                continue

            # Read in all TeX. Once definition detection is finished, all the TeX will be searched
            # for references to the defined terms.
            tex_by_file = file_utils.read_tex(arxiv_id)

            yield DetectDefinitionsTask(arxiv_id, sentences, tex_by_file)

    def process(
        self, item: DetectDefinitionsTask
    ) -> Iterator[Union[Definiendum, Definition, TermReference]]:
        sentences_ordered = sorted(item.sentences, key=lambda s: s.start)
        num_sentences = len(sentences_ordered)
        end_position_of_last_sentence = sentences_ordered[-1].end

        if len(item.sentences) == 0:
            logging.warning(  # pylint: disable=logging-not-lazy
                "No sentences found for arXiv ID %s. Skipping detection of sentences "
                + "that contain entities.",
                item.arxiv_id,
            )
            return

        # Load the pre-trained definition detection model.
        prediction_type = "DocDef2+AI2020+W00"
        model = DefinitionDetectionModel(prediction_type)

        definition_index = 0
        features = []
        sentences: List[EmbellishedSentence] = []

        definiendums: Dict[TermName, List[Definiendum]] = defaultdict(list)
        term_phrases: List[str] = []
        abbreviations: List[str] = []
        symbol_nicks: List[str] = []
        definitions: Dict[DefinitionId, Definition] = {}

        with tqdm(
            total=num_sentences, disable=(not self.args.show_progress)
        ) as progress:

            for sentence_index, sentence in enumerate(sentences_ordered):
                progress.update(1)

                # Only attempt to process sentences that have been marked as likely to be proper
                # plaintext. Note that this means some sentences may be skipped that didn't pass
                # heuristics in the sentence extractor.
                if not sentence.validity_guess:
                    continue

                # Extract features from raw text.
                featurized_text = model.featurize(sentence.legacy_definition_input)
                features.append(featurized_text)
                sentences.append(sentence)

                # Process sentences in batches.
                if (
                    len(features) >= self.args.batch_size
                    or sentence_index == num_sentences - 1
                ):

                    # Detect terms and definitions in each sentence with a pre-trained definition
                    # extraction model, from the featurized text.

                    (_, slots, slots_confidence) = model.predict_batch(
                        cast(List[Dict[Any, Any]], features)
                    )

                    # Package extracted terms and definitions into a representation that's
                    # easier to process.
                    for (
                        s,
                        sentence_features,
                        termdef_sentence_slots,
                        termdef_sentence_slots_confidence,
                        abbrexp_sentence_slots,
                        abbrexp_sentence_slots_confidence,
                        symnick_sentence_slots,
                        symnick_sentence_slots_confidence,
                    ) in zip(
                        sentences,
                        features,
                        slots["W00"],
                        slots_confidence["W00"],
                        slots["AI2020"],
                        slots_confidence["AI2020"],
                        slots["DocDef2"],
                        slots_confidence["DocDef2"],
                    ):
                        # Extract TeX for each symbol from a parallel representation of the
                        # sentence, so that the TeX for symbols can be saved.
                        # Types of [term and definition] pairs.
                        #   [nickname and definition] for symbols.
                        #   [abbreviation and expansion] for abbreviations.
                        #   [term and definition] for other types.
                        symbol_texs = get_symbol_texs(
                            s.legacy_definition_input, s.with_formulas_marked
                        )

                        # Only process slots when they include both 'TERM' and 'DEFINITION'.
                        if (
                            "TERM" not in termdef_sentence_slots
                            or "DEF" not in termdef_sentence_slots
                        ):
                            term_definition_pairs = []
                        else:
                            term_definition_pairs = consolidate_keyword_definitions(
                                s.legacy_definition_input,
                                sentence_features["tokens"],
                                termdef_sentence_slots,
                                termdef_sentence_slots_confidence,
                                "W00",
                            )

                        if (
                            "TERM" not in abbrexp_sentence_slots
                            or "DEF" not in abbrexp_sentence_slots
                        ):
                            abbreviation_expansion_pairs = []
                        else:
                            abbreviation_expansion_pairs = consolidate_keyword_definitions(
                                s.legacy_definition_input,
                                sentence_features["tokens"],
                                abbrexp_sentence_slots,
                                abbrexp_sentence_slots_confidence,
                                "AI2020",
                            )

                        if (
                            "TERM" not in symnick_sentence_slots
                            or "DEF" not in symnick_sentence_slots
                        ):
                            symbol_nickname_pairs = []
                        else:
                            symbol_nickname_pairs = consolidate_keyword_definitions(
                                s.legacy_definition_input,
                                sentence_features["tokens"],
                                symnick_sentence_slots,
                                symnick_sentence_slots_confidence,
                                "DocDef2",
                            )

                        pairs = (
                            term_definition_pairs
                            + symbol_nickname_pairs
                            + abbreviation_expansion_pairs
                        )
                        for pair in pairs:
                            tex_path = s.tex_path
                            definiendum_id = (
                                f"definiendum-{tex_path}-{definition_index}"
                            )
                            definition_id = f"definition-{tex_path}-{definition_index}"
                            definiendum_text = pair.term_text
                            definiendum_type = pair.term_type
                            definition_type = pair.definition_type

                            definiendum_confidence = pair.term_confidence
                            definition_confidence = pair.definition_confidence

                            # Map definiendum and definition start and end positions back to
                            # their original positions in the TeX.
                            offsets = s.legacy_definition_input_journal.initial_offsets(
                                pair.term_start, pair.term_end
                            )
                            if offsets[0] is None or offsets[1] is None:
                                logging.warning(  # pylint: disable=logging-not-lazy
                                    "Could not find offsets of definiendum %s in original TeX "
                                    + "(from sentence %s, file %s, arXiv ID %s). Definiendum will not be saved.",
                                    pair.term_text,
                                    s.id_,
                                    s.tex_path,
                                    item.arxiv_id,
                                )
                                continue
                            definiendum_start = s.start + offsets[0]
                            definiendum_end = s.start + offsets[1]

                            offsets = s.legacy_definition_input_journal.initial_offsets(
                                pair.definition_start, pair.definition_end
                            )
                            if offsets[0] is None or offsets[1] is None:
                                logging.warning(  # pylint: disable=logging-not-lazy
                                    "Could not find offsets of definition %s in original TeX "
                                    + "(from sentence %s, file %s, arXiv ID %s). Definiendum will not be saved.",
                                    pair.definition_text,
                                    s.id_,
                                    s.tex_path,
                                    item.arxiv_id,
                                )
                                continue
                            definition_start = s.start + offsets[0]
                            definition_end = s.start + offsets[1]

                            # Extract document-level features from sentence.
                            position_ratio = (
                                definiendum_start / end_position_of_last_sentence
                            )
                            section_name = s.section_name

                            try:
                                tex = item.tex_by_file[tex_path]
                            except KeyError:
                                logging.warning(  # pylint: disable=logging-not-lazy
                                    "Could not find TeX for %s. TeX will not be included in "
                                    + "the output data for definition '%s' for term '%s'",
                                    tex_path,
                                    pair.definition_text,
                                    definiendum_text,
                                )
                                definiendum_tex = "NOT AVAILABLE"
                                definition_tex = "NOT AVAILABLE"
                            else:
                                if (
                                    definiendum_type == "symbol"
                                    and symbol_texs is not None
                                    and pair.term_start in symbol_texs
                                ):
                                    definiendum_tex = symbol_texs[pair.term_start]
                                    definiendum_text = definiendum_tex
                                else:
                                    definiendum_tex = tex.contents[
                                        definiendum_start:definiendum_end
                                    ]
                                definition_tex = tex.contents[
                                    definition_start:definition_end
                                ]

                            # Save the definition to file.
                            definition = Definition(
                                id_=definition_id,
                                start=definition_start,
                                end=definition_end,
                                definiendum=definiendum_text,
                                type_=definition_type,
                                tex_path=tex_path,
                                tex=definition_tex,
                                text=pair.definition_text,
                                context_tex=sentence.context_tex,
                                sentence_id=sentence.id_,
                                intent=True,
                                confidence=definition_confidence,
                            )
                            definitions[definition_id] = definition
                            yield definition

                            # Don't save the definiendum to file yet. Save it in memory first, and then
                            # save it to file once it's done being processed. It will need
                            # to be associated with other definitions. Also, other references
                            # to the term will be detected before this method is over.
                            definiendum = Definiendum(
                                id_=definiendum_id,
                                text=definiendum_text,
                                type_=definiendum_type,
                                confidence=definiendum_confidence,
                                # Link the definiendum to the text that defined it.
                                definition_id=definition_id,
                                # Because a term can be defined multiple places in the paper, these
                                # three lists of definition data will be filled out once all of the
                                # definitions have been found.
                                definition_ids=[],
                                definitions=[],
                                definition_texs=[],
                                sources=[],
                                start=definiendum_start,
                                end=definiendum_end,
                                tex_path=tex_path,
                                tex=definiendum_tex,
                                context_tex=sentence.context_tex,
                                sentence_id=sentence.id_,
                                # Document-level features below.
                                position_ratio=position_ratio,
                                position_ratios=[],
                                section_name=section_name,
                                section_names=[],
                            )
                            definiendums[definiendum_text].append(definiendum)
                            if definiendum.type_ == "term":
                                term_phrases.append(definiendum.text)
                            if definiendum.type_ == "abbreviation":
                                abbreviations.append(definiendum.text)
                            if definiendum.type_ == "symbol":
                                symbol_nicks.append(definiendum.text)

                            definition_index += 1

                    features = []
                    sentences = []

        logging.debug(  # pylint: disable=logging-not-lazy
            "Finished detecting definitions for paper %s. Now finding references to defined terms.",
            item.arxiv_id,
        )

        all_definiendums: List[Definiendum] = []
        for _, definiendum_list in definiendums.items():
            all_definiendums.extend(definiendum_list)

        definition_ids: Dict[TermName, List[DefinitionId]] = {}
        definition_texs: Dict[TermName, List[str]] = {}
        definition_texts: Dict[TermName, List[str]] = {}
        sources: Dict[TermName, List[str]] = {}
        position_ratios: Dict[TermName, List[float]] = {}
        section_names: Dict[TermName, List[str]] = {}

        # Associate terms with all definitions that apply to them.
        for term, definiendum_list in definiendums.items():
            definition_ids[term] = [
                definiendum.definition_id for definiendum in definiendum_list
            ]
            definition_texs[term] = [
                definitions[definiendum.definition_id].tex
                for definiendum in definiendum_list
            ]
            definition_texts[term] = [
                definitions[definiendum.definition_id].text
                for definiendum in definiendum_list
            ]
            sources[term] = ["model"] * len(definition_ids[term])
            position_ratios[term] = [
                definiendum.position_ratio for definiendum in definiendum_list
            ]
            section_names[term] = [
                definiendum.section_name
                for definiendum in definiendum_list
                if definiendum.section_name is not None
            ]

        # Associate each definiendum with all applicable definitions, and save them to file.
        for _, definiendum_list in definiendums.items():
            for definiendum in definiendum_list:
                definiendum.definition_ids.extend(definition_ids[definiendum.text])
                definiendum.definition_texs.extend(definition_texs[definiendum.text])
                definiendum.definitions.extend(definition_texts[definiendum.text])
                definiendum.sources.extend(sources[definiendum.text])
                definiendum.position_ratios.extend(position_ratios[definiendum.text])
                definiendum.section_names.extend(section_names[definiendum.text])
                yield definiendum

        # Detect all other references to the defined terms. Detect references to textual
        # terms and abbreviations. References to symbols need not be found here; they
        # will be detected automatically in the symbol extraction code.
        term_index = 0

        for tex_path, file_contents in item.tex_by_file.items():
            term_extractor = PhraseExtractor(
                term_phrases + abbreviations
            )
            for t in term_extractor.parse(tex_path, file_contents.contents):

                # Don't save term references if they are already in the definiendums.
                if any([overlaps(definiendum, t) for definiendum in all_definiendums]):
                    continue

                logging.debug(
                    "Found reference to term %s at (%d, %d) in %s for arXiv ID %s",
                    t.text,
                    t.start,
                    t.end,
                    t.tex_path,
                    item.arxiv_id,
                )
                type_ = (
                    "abbreviation"
                    if t.text in abbreviations
                    else "term"
                    if t.text in term_phrases
                    else "symbol"
                    if t.text in symbol_nicks
                    else "unknown"
                )
                yield TermReference(
                    id_=f"term-{t.tex_path}-{term_index}",
                    text=t.text,
                    type_=type_,
                    definition_ids=definition_ids[t.text],
                    definitions=definition_texts[t.text],
                    definition_texs=definition_texs[t.text],
                    sources=sources[t.text],
                    position_ratios=position_ratios[t.text],
                    section_names=section_names[t.text],
                    start=t.start,
                    end=t.end,
                    tex_path=t.tex_path,
                    tex=t.tex,
                    context_tex=t.context_tex,
                )
                term_index += 1

    def save(
        self,
        item: DetectDefinitionsTask,
        result: Union[Definiendum, Definition, TermReference],
    ) -> None:

        output_dir = directories.arxiv_subdir("detected-definitions", item.arxiv_id)
        if not os.path.exists(output_dir):
            os.makedirs(output_dir)

        definiendums_path = os.path.join(output_dir, "entities-definiendums.csv")
        definitions_path = os.path.join(output_dir, "entities-definitions.csv")
        term_references_path = os.path.join(output_dir, "entities-term-references.csv")

        if isinstance(result, Definiendum):
            file_utils.append_to_csv(definiendums_path, result)
        elif isinstance(result, Definition):
            file_utils.append_to_csv(definitions_path, result)
        elif isinstance(result, TermReference):
            file_utils.append_to_csv(term_references_path, result)


if __name__ == "__main__":
    example = """30373
25512
65332
33549
35390"""

    example_input = example.split("\n")

    with open("input/day8.txt") as input:
        problem_input = [i.strip() for i in input.readlines()]

    tree_map = [[int(t) for t in line] for line in problem_input]
    num_rows = len(tree_map)
    num_cols = len(tree_map[0])

    # precompute the columns
    tree_cols = [[tree_map[r][c] for r in range(num_rows)] for c in range(num_cols)]

    visible_trees = 2 * num_rows + 2 * num_cols - 4  # de-double-count the corners
    best_scenic_score = 0

    for row in range(1, num_rows - 1):
        tree_row = tree_map[row]

        for col in range(1, num_cols - 1):
            tree_col = tree_cols[col]
            tree = tree_map[row][col]

            # Part 1
            visible_from_left = all(t < tree for t in tree_row[0:col])
            visible_from_right = all(t < tree for t in tree_row[col + 1 :])
            visible_from_top = all(t < tree for t in tree_col[0:row])
            visible_from_bottom = all(t < tree for t in tree_col[row + 1 :])

            if any(
                (
                    visible_from_left,
                    visible_from_right,
                    visible_from_top,
                    visible_from_bottom,
                )
            ):
                visible_trees += 1

            # Part 2
            left_score = 0

            for t in tree_row[col - 1 :: -1]:
                left_score += 1
                if t >= tree:
                    break

            right_score = 0

            for t in tree_row[col + 1 :]:
                right_score += 1
                if t >= tree:
                    break

            up_score = 0

            for t in tree_col[row - 1 :: -1]:
                up_score += 1
                if t >= tree:
                    break

            down_score = 0

            for t in tree_col[row + 1 :]:
                down_score += 1
                if t >= tree:
                    break

            scenic_score = left_score * right_score * up_score * down_score

            if scenic_score > best_scenic_score:
                best_scenic_score = scenic_score

    print(f"part 1: visible trees: {visible_trees}")
    print(f"part 2: best scenic score: {best_scenic_score}")


# -*- coding: utf-8 -*-
#harvest thesis from UMass Amherst
#FS: 2022-11-10

import getopt
import sys
import os
import urllib.request, urllib.error, urllib.parse
from bs4 import BeautifulSoup
import re
import ejlmod3
import time
import undetected_chromedriver as uc

publisher = 'UMass Amherst'
jnlfilename = 'THESES-UMassAmherst-%s' % (ejlmod3.stampoftoday())

pages = 1
years = 2
skipalreadyharvested = True
boring = ['Afro-American Studies', 'Animal Biotechnology & Biomedical Sciences', 'Anthropology',
          'Biomedical Engineering', 'Chemical Engineering', 'Chemistry', 'Civil and Environmental Engineering',
          'Civil Engineering', 'Communication Disorders', 'Communication', 'Comparative Literature',
          'Economics', 'Education', 'Electrical and Computer Engineering', 'English',
          'Environmental Conservation', 'Food Science', 'Geosciences', 'Hispanic Literatures & Linguistics',
          'History', 'Hospitality & Tourism Management', 'Industrial Engineering & Operations Research',
          'Kinesiology', 'Linguistics', 'Management', 'Microbiology', 'Molecular and Cellular Biology',
          'Music', 'Neuroscience and Behavior', 'Nursing', 'Organismic and Evolutionary Biology', 'Philosophy',
          'Plant Biology', 'Political Science', 'Polymer Science and Engineering', 'Psychology',
          'Public Health', 'Regional Planning', 'Resource Economics', 'School Psychology', 'Sociology',
          'Wildlife & Fisheries Conservation', 'American Studies', 'Biochemistry', 'Biology',
          'Education (also CAGS)', 'Education; Children and Family Studies', 'Isenberg School of Management',
          'Education; Language, Literacy and Culture', 'Educational Policy and Leadership',
          'Education Policy, Research and Administration; Educational Policy and Leadership',
          'Education; Teacher Education & Curriculum Studies', 'English - American Studies', 'Entomology',
          'Forest Resources', 'German and Scandinavian Studies', 'Germanic Languages & Literatures',
          'Hispanic Literatures and Linguistics', 'Industrial Engineering and Operations Research',
          'Languages, Literatures, and Cultures; Hispanic Literatures and Linguistics',
          'Lusophone Literatures and Cultures', 'Marine Sciences and Technology',
          'Mechanical and Industrial Engineering', 'Mechanical Engineering', 'Plant and Soil Sciences',
          'Plant, Soil & Insect Sciences', 'Psychology; Clinical Psychology',
          'Psychometric Methods, Education Statistics and Research Methods', 'Sport Management',
          'Wildlife and Fisheries Conservation', 'Animal Biotechnology and Biomedical Sciences',
          'Biostatistics and Epidemiology', 'Clinical Psychology',
          'Educational Planning, Research, and Administration', 'Language, Literacy, & Culture',
          'Educational Policy and Leadership; International Education',
          'Educational Policy, Research, and Administration', 'Education; Educational Policy and Leadership',
          'Education, Educational Policy Research, and Administration Research',
          'Education; Education Policy, Research, and Administration', 'Education; Research and Evaluation Methods',
          'Education; Student Development', 'Education; Teacher Education & School Improvement',
          'Epidemiology and Biostatistics', 'Isenberg School of Management; Operations Management',
          'Mechanical and Industrial Engineering; Industrial Engineering and Operations Research',
          'Neuroscience & Behavior', 'Organismic and Evolutionary Biology; Entomology',
          'Research and Evaluating Methods', 'Social Justice Education',
          'Teacher Education and Curriculum Studies; Language, Literacy, & Culture',
          'Teacher Education and Curriculum Studies', 'Teacher Education']

basetocurl = 'https://scholarworks.umass.edu/dissertations_2/index.'
tocextension = 'html'

options = uc.ChromeOptions()
options.binary_location = '/usr/bin/google-chrome'
options.binary_location='/usr/bin/chromium'
options.add_argument('--headless')
chromeversion = int(re.sub('.*?(\d+).*', r'\1', os.popen('%s --version' % (options.binary_location)).read().strip()))
driver = uc.Chrome(version_main=chromeversion, options=options)

if skipalreadyharvested:
    alreadyharvested = ejlmod3.getalreadyharvested(jnlfilename)

prerecs = []
date = False
for i in range(pages):
    tocurl = basetocurl + tocextension
    ejlmod3.printprogress('=', [[i+1, pages], [tocurl]])
    try:
        tocpage = BeautifulSoup(urllib.request.build_opener(urllib.request.HTTPCookieProcessor).open(tocurl), features="lxml")
        time.sleep(3)
    except:
        print("retry %s in 180 seconds" % (tocurl))
        time.sleep(180)
    for div in tocpage.body.find_all('div'):
        if div.has_attr('class') and re.search('locks', div['class'][0]):
            for child in div.children:
                try:
                    name = child.name
                except:
                    continue
                if name == 'h4':
                    for span in child.find_all('span'):
                        date = span.text.strip()
                elif name == 'p':
                    #year = int(re.sub('.*(20\d\d).*', r'\1', rec['date']))
                    if int(date) > ejlmod3.year(backwards=years):
                        if child.has_attr('class') and 'article-listing' in child['class']:
                            rec = {'jnl' : 'BOOK', 'tc' : 'T', 'date' : date, 'note' : []}
                            for a in child.find_all('a'):
                                rec['tit'] = a.text.strip()
                                rec['artlink'] = a['href']
                                a.replace_with('')
                            if ejlmod3.checkinterestingDOI(rec['artlink']):
                                prerecs.append(rec)
    print('  ', len(prerecs))
    tocextension = '%i.html' % (i+2)

i = 0
recs = []
for rec in prerecs:
    keepit = True
    i += 1
    ejlmod3.printprogress('-', [[i, len(prerecs)], [rec['artlink']], [len(recs)]])
    try:
        artpage = BeautifulSoup(urllib.request.build_opener(urllib.request.HTTPCookieProcessor).open(rec['artlink']), features="lxml")
        time.sleep(3)
    except:
        print("retry %s in 180 seconds" % (rec['artlink']))
        time.sleep(180)
        artpage = BeautifulSoup(urllib.request.build_opener(urllib.request.HTTPCookieProcessor).open(rec['artlink']), features="lxml")
    ejlmod3.metatagcheck(rec, artpage, ['description', 'bepress_citation_author', 'bepress_citation_pdf_url', 'bepress_citation_doi',
                                        'bepress_citation_online_date'])
    if 'doi' in rec:
        rec['doi'] = re.sub('.*doi.org\/', '', rec['doi'])
    #supervisor
    for div in artpage.body.find_all('div', attrs = {'id' : 'advisor1'}):
        for p in div.find_all('p'):
            rec['supervisor'] = [[ re.sub('^Dr. ', '', p.text.strip()) ]]
    for div in artpage.body.find_all('div', attrs = {'id' : 'advisor2'}):
        for p in div.find_all('p'):
            rec['supervisor'].append( [re.sub('^Dr. ', '', p.text.strip())] )
    #ORCID
    for div in artpage.body.find_all('div', attrs = {'id' : 'orcid'}):
        for h4 in div.find_all('h4'):
            if re.search('uthor', h4.text):
                for p in div.find_all('p'):
                    if re.search('orcid', p.text):
                        rec['autaff'][-1].append(re.sub('.*orcid.*?\/', 'ORCID:', p.text.strip()))
                    else:
                        rec['autaff'][-1].append('ORCID:'+p.text.strip())
                if rec['autaff'][0][0] == ',':
                    orcidlink = re.sub('ORCID:', 'https://orcid.org/', rec['autaff'][-1][-1])
                    print('   try to get author name from %s' % (orcidlink))
                    driver.get(orcidlink)
                    orcidpage = BeautifulSoup(driver.page_source, features="lxml")
                    author = ''
                    for meta in orcidpage.head.find_all('meta', attrs = {'property' : 'profile:last_name'}):
                        author = meta['content']
                    for meta in orcidpage.head.find_all('meta', attrs = {'property' : 'profile:first_name'}):
                        author += ', ' + meta['content']
                    if author:
                        rec['autaff'][0][0] = author
                        print('\n   got author name "%s" from ORCID page\n' % (author))
                    else:
                        rec['autaff'][0][0] = 'Doe, John'
                        print('\n   could not get author name from ORCID page\n')
                    time.sleep(10)
            else:
                print(div)
    #keywords
    for div in artpage.body.find_all('div', attrs = {'id' : 'bp_categories'}):
        for p in div.find_all('p'):
            rec['keyw'] = re.split(' *\| *', p.text.strip())
    #department
    for div in artpage.body.find_all('div', attrs = {'id' : 'department'}):
        for p in div.find_all('p'):
            department = p.text.strip()
            if department in boring:
                keepit = False
            elif department in ['Computer Science']:
                rec['fc'] = 'c'
            elif department in ['Astronomy']:
                rec['fc'] = 'a'
            elif department in ['Applied Mathematics', 'Mathematics']:
                rec['fc'] = 'm'
                rec['autaff'][-1].append('UMASS, Amherst, Dept. Math. Stat.')
            elif department in ['Statistics']:
                rec['fc'] = 's'
                rec['autaff'][-1].append('UMASS, Amherst, Dept. Math. Stat.')
            elif department in ['Physics and Astronomy', 'Physics & Astronomy', 'Physics']:
                rec['autaff'][-1].append('Massachusetts U., Amherst')
            else:
                rec['note'].append(department)
    #license
    for link in artpage.find_all('link', attrs = {'rel' : 'license'}):
        rec['license'] = {'url' : link['href']}
    if keepit:
        if len(rec['autaff'][-1]) == 1:
            rec['autaff'][-1].append(publisher)
        if skipalreadyharvested and 'doi' in rec and rec['doi'] in alreadyharvested:
            print('   %s already in backup' % (rec['doi']))
        else:
            ejlmod3.printrecsummary(rec)
            recs.append(rec)
    else:
        ejlmod3.adduninterestingDOI(rec['artlink'])

ejlmod3.writenewXML(recs, publisher, jnlfilename)


# Programmed by Scar
"""
设计模式
    设计模式是前人工作的总结和提炼，通常，被人们广泛流传的设计模式都是针对某一特定问题的成熟的解决方案
    使用设计模式是为了可重复代码、让代码更容易被他人理解、保证代码可靠性
单例设计模式
    魔都：让类创建的对象，在系统中只有唯一的一个实例
    每次执行类名()返回得分对象，内存地址是相同的
"""

class MusicPlayer(object):
    # 记录第一个被创建对象的引用
    instance = None

    def __new__(cls, *args, **kwargs):
        # 1. 判断类属性是否是空对象
        if cls.instance is None:
            # 2. 调用父类的方法，为第一个对象分配空间
            cls.instance = super().__new__(cls)

        # 3. 返回类属性保存的对象引用
        return cls.instance

# 创建多个对象
player1 = MusicPlayer()
print(player1)

player2 = MusicPlayer()
print(player2)


import unittest
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord
from crispr_scoring_module import CRISPRScoringModule

class TestCRISPRScoringModule(unittest.TestCase):

    def setUp(self):
        self.fasta_sequences = [
            SeqRecord(Seq("AAACCCGGGTTT"), id="seq1"),
            SeqRecord(Seq("GGGCCCAAATTTCGG"), id="seq2")
        ]
        self.pam_sequence = 'CGG'

        # Initialize CRISPRScoringModule with the test data
        self.crispr_tool = CRISPRScoringModule(None, self.pam_sequence)
        self.crispr_tool.target_sites = [
            {'start': 5, 'end': 7, 'sequence': 'CGG', 'score': None},
            {'start': 12, 'end': 14, 'sequence': 'CGG', 'score': None}
        ]

    def test_find_target_sites(self):
        # Here you can test the find_target_sites method.
        # Since this method needs to read from a FASTA file, you can create a temporary FASTA file
        # or mock the file reading part using Python's unittest.mock.
        # It is generally a good practice to use mocking for file I/O in unit tests.
        pass

    def test_score_target_sites(self):
        # Test scoring based on GC content
        self.crispr_tool.score_target_sites()

        # Check the scores
        for target_site in self.crispr_tool.target_sites:
            sequence = target_site['sequence']
            expected_score = 100.0 * sequence.count('G') + sequence.count('C') / len(sequence)
            self.assertEqual(target_site['score'], expected_score)

if __name__ == '__main__':
    unittest.main()


"""Preprocessing imagenet2012 dataset into TFRecord format

This code is adapted from
https://github.com/google-research/big_vision/blob/main/big_vision/tools/download_tfds_datasets.py
"""

from absl import app
import tensorflow_datasets as tfds
import resource

low, high = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE, (high, high))

def main(argv):
  if len(argv) > 1 and "download_tfds_datasets.py" in argv[0]:
    datasets = argv[1:]
  else:
    exit(0)
  for d in datasets:
    tfds.load(name=d, download=True, data_dir='')

if __name__ == "__main__":
  app.run(main)


class MaJudge:
	def __init__(self):
		self.signal = ''
		self.candle = ''
		self.ma = ''
		self.ma_buffer = ''
		self.hige_buffer = ''
		self.position = ''

	def set_params(self, signal, candle, ma, ma_buffer, hige_buffer, position):
		self.signal = signal
		self.candle = candle
		self.ma = ma
		self.ma_buffer = ma_buffer
		self.hige_buffer = hige_buffer
		self.position = position

	# 売買判定する関数
	def judge_open_signal(self):
		signal = self.signal
		candle = self.candle
		ma = self.ma
		ma_buffer = self.ma_buffer
		hige_buffer = self.hige_buffer

		# 移動平均線よりも下に位置してる
		if candle["open_price"] < candle["{}ma".format(ma)]:
			# 移動平均線を上抜ける
			if candle["close_price"] > candle["{}ma".format(ma)] + ma_buffer:
				# 上ひげが長い時は入らない
				if candle["high_price"] - candle["close_price"] < hige_buffer:
					# 上に移動平均線がいっぱいある時は入らない
					# if candle["close_price"] > candle["20ma"]:
					return signal.set_buy(candle["close_price"])

		# 移動平均線よりも上に位置してる
		if candle["open_price"] > candle["{}ma".format(ma)]:
			# 移動平均線を下抜ける
			if candle["close_price"] < candle["{}ma".format(ma)] - ma_buffer:
				# 下ひげが長い時は入らない
				if candle["close_price"] - candle["low_price"] < hige_buffer:
					# 上に移動平均線がいっぱいある時は入らない
					# if candle["close_price"] < candle["20ma"]:
					return signal.set_sell(candle["close_price"])

		signal.reset()

	def judge_close_signal(self):
		signal =self.signal
		candle =self.candle
		position =self.position

		# 買いのポジションが入ってる
		if position.is_buy():
			# ろうそく足が赤
			if candle["open_price"] > candle["close_price"]:
				return signal.set_sell(candle["close_price"])

		# 売りのポジションが入ってる
		if position.is_sell():
			# ろうそく足が緑
			if candle["open_price"] < candle["close_price"]:
				return signal.set_buy(candle["close_price"])


import xarray as xr
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('--GCM', type=str, required=True)
parser.add_argument('--scen', type=str, required=True)
parser.add_argument('--var', type=str, required=True)

args = parser.parse_args()

def reorder(GCM,scen,var):
    ds = xr.open_dataset(var+'/'+GCM+'_'+scen+'_r1i1p1_r240x120-MRNBC-AWAP_'+
                         var+'_1960-2100.nc')

    ds = ds.drop_dims('bnds').transpose('lat','lon','time')
    print('reordered')
    ds_chunked = ds.chunk({'time': 1})
    ds_chunked.to_netcdf(var+'/'+GCM+'_'+scen+'_r1i1p1_'+var+'_reorder.nc')
    print('wrote file')

reorder(args.GCM,args.scen,args.var)


from django.contrib.auth.decorators import login_required
from django.contrib.auth.forms import UserCreationForm
from django.contrib.auth.views import LoginView, LogoutView
from django.shortcuts import render
from django.urls import reverse_lazy
from django.views.generic import CreateView

signup = CreateView.as_view(
    form_class=UserCreationForm,
    template_name="partials/form.html",
    extra_context={"form_name": "Sign up", "submit_label": "Sign up"},
    success_url=reverse_lazy("accounts:login"),
)

login = LoginView.as_view(
    template_name="partials/form.html",
    extra_context={"form_name": "로그인", "submit_label": "로그인"},
)

logout = LogoutView.as_view(next_page="accounts:login")

@login_required
def profile(request):
    user = request.user
    return render(
        request,
        "accounts/profile.html",
        {
            "user": user,
        },
    )


"""
leetcode No.208
字典树，用于快速的字符串查找
"""

class Trie:

    def __init__(self):
        """
        Initialize your data structure here.
        """
        self.root = dict()

    def insert(self, word: str) -> None:
        """
        Inserts a word into the trie.
        """
        p = self.root
        for x in word:
            if x not in p:
                p[x] = dict()
            p = p[x]
        p[-1] = True

    def search(self, word: str) -> bool:
        """
        Returns if the word is in the trie.
        """
        p = self.root
        for i in range(len(word)):
            next_node = p.get(word[i], None)
            if not next_node:
                return False
            p = next_node
        return True if p.get(-1, None) else False

    def startsWith(self, prefix: str) -> bool:
        """
        Returns if there is any word in the trie that starts with the given prefix.
        """
        p = self.root
        for i in range(len(prefix)):
            next_node = p.get(prefix[i], None)
            if not next_node:
                return False
            p = next_node
        return True


#!/bin/python3

import math
import os
import random
import re
import sys
from collections import deque

def median(buckets, d):
    s = 0
    last = 0

    for (val, ctr) in enumerate(buckets):
        if ctr > 0:
            s += ctr
            if s > d//2 and d % 2 == 1:
                return val
            elif s > d//2 and d % 2 == 0:
                if ctr >= 2:
                    return val / 2
                else:
                    return (last + val) / 2

            last = val

def activityNotifications(ex, d):
    ret = 0
    buckets = [0] * 300
    queue = deque()

    for (i,x) in enumerate(ex):
        if i >= d:
            med = median(buckets,d)
            if (x >= med * 2):
                print(buckets)
                print(med)
                print(i, x)
                return 0
            ret += (x >= median(buckets,d) * 2)
            y = queue.popleft()
            buckets[y] -= 1

        buckets[x] += 1
        queue.append(x)

    return ret

if __name__ == '__main__':
    #fptr = open(os.environ['OUTPUT_PATH'], 'w')
    nd = input().split()
    n = int(nd[0])
    d = int(nd[1])
    expenditure = list(map(int, input().rstrip().split()))
    result = activityNotifications(expenditure, d)
    #fptr.write(str(result) + '\n')

    #fptr.close()
    print(result)


import struct

from mil_usb_to_can.sub8 import ApplicationPacket

# CAN ID for the channel from MOBO to actuator board
SEND_ID = 0x51

class CommandMessage(ApplicationPacket):
    """
    Represents a packet sent from the motherboard to the actuator board. Being a packet,
    inherits from :class:`ApplicationPacket`.

    Attributes:
        IDENTIFIER (int): The ASCII value of "A".
        NUM_VALVES (int): The number of valves. Set to ``12`` currently.
    """

    IDENTIFIER = ord("A")
    NUM_VALVES = 12

    @property
    def address(self) -> int:
        """
        The integer address of this command.

        :rtype: int
        """
        return struct.unpack("B", self.payload[0])[0]

    @property
    def write(self) -> bool:
        """
        If True, this message sets the valve open or closed. If False, this
        message requests the status of a valve.

        :rtype: bool
        """
        return struct.unpack("B", self.payload[1])[0] == 1

    @property
    def on(self) -> bool:
        """
        If True, this message commanded the valve to be open. Else, the valve
        is commanded to be closed.

        This parameter only has any effect if :attr:`~.write` is true.

        :rtype: bool
        """
        return struct.unpack("B", self.payload[2])[0] == 1

    @classmethod
    def create_command_message(cls, address: int = 0, write: int = 0, on: int = 0):
        """
        Create a new command message.

        Args:
            address (int): Address of valve, ranging from 0-11.
            write (int): If true, set valve open or close. If False, request
              the status of the specified valve.
            on (int): If true, command valve to open. If write if False, this has no effect

        Raises:
            InvalidAddressException: The valve address is not valid, and is outside
              of the 0-11 range.
        """
        if address < 0 or address >= cls.NUM_VALVES:
            raise InvalidAddressException(address)
        write_byte = 1 if write else 0
        on_byte = 1 if on else 0
        payload = struct.pack("BBB", address, write_byte, on_byte)
        return cls(cls.IDENTIFIER, payload)

    def __str__(self):
        return "CommandMessage(address={}, write={}, on={})".format(
            self.address,
            self.write,
            self.on,
        )

class InvalidAddressException(RuntimeError):
    """
    Exception noting that the requested valve address is outside of the range available
    on the sub. Inherits from :class:`RuntimeError`.
    """

    def __init__(self, address: int):
        super().__init__(
            "Attempted to command valve {}, but valid addresses are only [0,{}]".format(
                address,
                CommandMessage.NUM_VALVES - 1,
            ),
        )

class FeedbackMessage(ApplicationPacket):
    """
    Represents a message received from the actuator board sent to the motherboard

    Attributes:
        IDENITIFER (int): The packet identifier. Set to the ASCII value of "A".
    """

    IDENTIFIER = ord("A")

    @property
    def address(self) -> int:
        """
        The address of valve referenced by this command. Should be between 0-11.

        :rtype: int
        """
        return struct.unpack("B", self.payload[0])[0]

    @property
    def on(self) -> bool:
        """
        If True, valve is currently open. If False, valve is currently closed.

        :rtype: bool
        """
        return struct.unpack("B", self.payload[1])[0] == 1

    @classmethod
    def create_feedback_message(cls, address: int = 0, on: bool = False):
        """
        Create a feedback message.

        Args:
            address (int): valve address this status references, integer 0-11
            on (bool): If True, valve is open
        """
        on_byte = 1 if on else 0
        payload = struct.pack("BBB", address, on_byte, 0)
        return cls(cls.IDENTIFIER, payload)

    def __str__(self):
        return f"FeedbackMessage(address={self.address}, on={self.on})"


import csv
import os.path
import DeepLearning.MSResnet
import DeepLearning.experiments
import pklDataset
import matplotlib.pyplot as plt
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.distributed as dist
import argparse

import torchvision
import torchvision.transforms as transforms

from Insight.insightface.recognition.arcface_torch.backbones import get_model
from Insight.insightface.recognition.arcface_torch.dataset import MXFaceDataset, SyntheticDataset, DataLoaderX
from Insight.insightface.recognition.arcface_torch.partial_fc import PartialFC
from Insight.insightface.recognition.arcface_torch.utils.utils_config import get_config
from Insight.insightface.recognition.arcface_torch.utils.utils_logging import AverageMeter, init_logging
import logging

from prettytable import PrettyTable

# Method for calculating how many parameters were used in the network
def count_parameters(model):
    table = PrettyTable(["Modules", "Parameters"])
    total_params = 0
    for name, parameter in model.named_parameters():
        if not parameter.requires_grad: continue
        param = parameter.numel()
        table.add_row([name, param])
        total_params += param
    print(table)
    print(f"Total Trainable Params: {total_params}")
    return total_params

def main(args):
    # Define batch size for stochastic gradient descent

    fields = {}
    csv_name = r'accuracy_chart.csv'
    batch_size = 256
    filter_size = 3
    epochs = 60
    dropout_rate = 0.05
    num_of_filters = 32
    num_of_filters_2 = 64
    num_of_filters_3 = 32
    num_of_filters_4 = 32
    learning_rate = 0.001
    weight_decay = 0
    pooling_size = 2
    momentum = 0.9
    num_of_fc = 1
    test_name = '3X3 filter'

    fields['Batch Size'] = batch_size
    fields['Dropout Rate'] = dropout_rate
    fields['Filter Size'] = filter_size
    fields['Filter Count'] = num_of_filters
    fields['Filter Count 2'] = num_of_filters_2
    fields['Filter Count 3'] = num_of_filters_3
    fields['Filter Count 4'] = num_of_filters_4
    fields['Epochs'] = epochs
    fields['Learning Rate'] = learning_rate
    fields['Weight Decay'] = 0
    fields['Pooling Size'] = pooling_size
    fields['Number of FC'] = num_of_fc

    # Define directory to save dataset images
    data_directory = './data'
    pkl_path1 = 'Insight/insightface/recognition/arcface_torch/r18_features'
    pkl_path2 = 'Insight/insightface/recognition/arcface_torch/r50_features'

    # Datasets

    pkl_dataset = pklDataset.PklEmbeddingsDataset(pkl_path1,pkl_path2)

    [train_set, val_set, test_set] = pklDataset.SplitDataset(pkl_dataset, n_labels=40000, ratios = [0.006 , 0.006])

    interp_train_set , interp_set = pklDataset.InterpolateDatasetRandom(train_set,pkl_dataset,min_interp_num=20, max_interp_num=30,interpolated_div=1)

    # train_nn_space = pklDataset.CreateMeanLabels(train_set, 'train_not_interp')
    #train_nn_space = pklDataset.CreateMeanLabels(train_set, 'train')
    val_nn_space = pklDataset.CreateNNLabels(val_set, 'val_new_split')
    test_nn_space = pklDataset.CreateNNLabels(test_set, 'test_new_split')

    features_loss_fns = [torch.nn.CosineSimilarity(), torch.nn.MSELoss()]
    features_loss_weights = [1,300]
    label_loss_fns = []
    label_loss_weights = []

    msresnet = DeepLearning.MSResnet.MSResNet(input_channel=1, layers=[1, 1, 1, 1],num_classes=len(pkl_dataset.ds1.classes),embedding_size=512)
    #
    # # Backbone configs
    # cfg = get_config(args.config)
    # try:
    #     world_size = int(os.environ['WORLD_SIZE'])
    #     rank = int(os.environ['RANK'])
    #     dist.init_process_group('nccl')
    # except KeyError:
    #     world_size = 1
    #     rank = 0
    #     dist.init_process_group(backend='nccl', init_method="tcp://127.0.0.1:12584", rank=rank, world_size=world_size)
    #
    # local_rank = args.local_rank
    # torch.cuda.set_device(local_rank)
    # os.makedirs(cfg.output, exist_ok=True)
    # init_logging(rank, cfg.output)
    #
    # backbone = get_model(cfg.network, dropout=0.0, fp16=cfg.fp16, num_features=cfg.embedding_size).to(local_rank)
    #
    # if cfg.resume:
    #     try:
    #         backbone_pth = os.path.join(cfg.output, "backbone.pth")
    #         backbone.load_state_dict(torch.load(backbone_pth, map_location=torch.device(local_rank)))
    #         if rank == 0:
    #             logging.info("backbone resume successfully!")
    #     except (FileNotFoundError, KeyError, IndexError, RuntimeError):
    #         if rank == 0:
    #             logging.info("resume fail, backbone init successfully!")
    #
    # backbone = torch.nn.parallel.DistributedDataParallel(
    #     module=backbone, broadcast_buffers=False, device_ids=[local_rank])
    # margin_softmax = losses.get_loss(cfg.loss)
    # module_partial_fc = PartialFC(
    #     rank=rank, local_rank=local_rank, world_size=world_size, resume=cfg.resume,
    #     batch_size=cfg.batch_size, margin_softmax=margin_softmax, num_classes=cfg.num_classes,
    #     sample_rate=cfg.sample_rate, embedding_size=cfg.embedding_size, prefix=cfg.output)

    #
    #
    #
    #
    # fit_res_msresnet = DeepLearning.experiments.cnn_experiment(model=msresnet, run_name="rs50_features", ds_train=interp_train_set,
    #                                                            ds_val=val_set, ds_test=test_set,
    #                                                            bs_train=batch_size, bs_val=batch_size, optimizer=None,
    #                                                            epochs=200, early_stopping=10,
    #                                                            filters_per_layer=[64, 128, 512],
    #                                                            layers_per_block=0, pool_every=4, hidden_dims=[],
    #                                                            checkpoints="./checkpoints/resnet_interp_emb",
    #                                                            lr=0.001, features_loss_fns=features_loss_fns,
    #                                                            features_loss_weights=features_loss_weights,
    #                                                            label_loss_fns=label_loss_fns,
    #                                                            label_loss_weights=label_loss_weights,
    #                                                            model_type="cnn", batches=None,
    #                                                            train_nn_space=val_nn_space,
    #                                                            val_nn_space=val_nn_space,
    #                                                            test_nn_space=test_nn_space
    #                                                            )
    #
    # print("Training MLP")
    # fit_res_mlp = DeepLearning.experiments.cnn_experiment(model=None, run_name="rs50_features", ds_train=interp_train_set,
    #                                                       ds_val=val_set, ds_test=test_set,
    #                                                       bs_train=batch_size, bs_val=batch_size, optimizer=None,
    #                                                       epochs=200, early_stopping=10,
    #                                                       filters_per_layer=[64, 128, 512],
    #                                                       layers_per_block=0, pool_every=4, hidden_dims=[],
    #                                                       checkpoints="./checkpoints/mlp_interp_emb",
    #                                                       lr=0.001, features_loss_fns=features_loss_fns,
    #                                                       features_loss_weights=features_loss_weights,
    #                                                       label_loss_fns=label_loss_fns,
    #                                                       label_loss_weights=label_loss_weights,
    #                                                       model_type="cnn", batches=None,
    #                                                       train_nn_space=val_nn_space,
    #                                                       val_nn_space=val_nn_space,
    #                                                       test_nn_space=test_nn_space
    #                                                       )
    #
    #
    # fit_res_msresnet = DeepLearning.experiments.cnn_experiment(model=msresnet, run_name="rs50_features", ds_train=train_set,
    #                                                            ds_val=val_set, ds_test=test_set,
    #                                                            bs_train=batch_size, bs_val=batch_size, optimizer=None,
    #                                                            epochs=200, early_stopping=10,
    #                                                            filters_per_layer=[64, 128, 512],
    #                                                            layers_per_block=0, pool_every=4, hidden_dims=[],
    #                                                            checkpoints="./checkpoints/resnet_emb",
    #                                                            lr=0.001, features_loss_fns=features_loss_fns,
    #                                                            features_loss_weights=features_loss_weights,
    #                                                            label_loss_fns=label_loss_fns,
    #                                                            label_loss_weights=label_loss_weights,
    #                                                            model_type="cnn", batches=None,
    #                                                            train_nn_space=val_nn_space,
    #                                                            val_nn_space=val_nn_space,
    #                                                            test_nn_space=test_nn_space
    #                                                            )
    #
    print("Training Pretrained model")
    model = torch.load(
        r'/media/hovav/1df0a4b3-8e8b-4fe3-bd7d-499ea780a5a8/home/liorshai/CGMproject/FaceRecognition/checkpoints/mlp_pretrained')
    fit_res_mlp = DeepLearning.experiments.cnn_experiment(model=model, run_name="rs50_features",
                                                          ds_train=train_set,
                                                          ds_val=val_set, ds_test=test_set,
                                                          bs_train=batch_size, bs_val=batch_size, optimizer=None,
                                                          epochs=150, early_stopping=10,
                                                          filters_per_layer=[64, 128, 512],
                                                          layers_per_block=0, pool_every=4, hidden_dims=[],
                                                          checkpoints="./checkpoints/mlp_trained_full_with_interp_25_p",
                                                          lr=0.00001, features_loss_fns=features_loss_fns,
                                                          features_loss_weights=features_loss_weights,
                                                          label_loss_fns=label_loss_fns,
                                                          label_loss_weights=label_loss_weights,
                                                          model_type="cnn", batches=None,
                                                          train_nn_space=val_nn_space,
                                                          val_nn_space=val_nn_space,
                                                          test_nn_space=test_nn_space
                                                          )

    fit_res_mlp = DeepLearning.experiments.cnn_experiment(model=None, run_name="rs50_features",
                                                          ds_train=interp_train_set,
                                                          ds_val=val_set, ds_test=test_set,
                                                          bs_train=batch_size, bs_val=batch_size, optimizer=None,
                                                          epochs=200, early_stopping=10,
                                                          filters_per_layer=[64, 128, 512],
                                                          layers_per_block=0, pool_every=4, hidden_dims=[],
                                                          checkpoints="./checkpoints/mlp_interp_50_p",
                                                          lr=0.001, features_loss_fns=features_loss_fns,
                                                          features_loss_weights=features_loss_weights,
                                                          label_loss_fns=label_loss_fns,
                                                          label_loss_weights=label_loss_weights,
                                                          model_type="cnn", batches=None,
                                                          train_nn_space=val_nn_space,
                                                          val_nn_space=val_nn_space,
                                                          test_nn_space=test_nn_space
                                                          )

    fit_res_mlp = DeepLearning.experiments.cnn_experiment(model=None, run_name="rs50_features",
                                                          ds_train=interp_train_set,
                                                          ds_val=val_set, ds_test=test_set,
                                                          bs_train=batch_size, bs_val=batch_size, optimizer=None,
                                                          epochs=50, early_stopping=10,
                                                          filters_per_layer=[64, 128, 512],
                                                          layers_per_block=0, pool_every=4, hidden_dims=[],
                                                          checkpoints="./checkpoints/mlp_pretrained_50_p",
                                                          lr=0.001, features_loss_fns=features_loss_fns,
                                                          features_loss_weights=features_loss_weights,
                                                          label_loss_fns=label_loss_fns,
                                                          label_loss_weights=label_loss_weights,
                                                          model_type="cnn", batches=None,
                                                          train_nn_space=val_nn_space,
                                                          val_nn_space=val_nn_space,
                                                          test_nn_space=test_nn_space
                                                          )

    print("Training Pretrained model")
    model = torch.load(
        r'/media/hovav/1df0a4b3-8e8b-4fe3-bd7d-499ea780a5a8/home/liorshai/CGMproject/FaceRecognition/checkpoints/mlp_pretrained_50_p')
    fit_res_mlp = DeepLearning.experiments.cnn_experiment(model=model, run_name="rs50_features",
                                                          ds_train=train_set,
                                                          ds_val=val_set, ds_test=test_set,
                                                          bs_train=batch_size, bs_val=batch_size, optimizer=None,
                                                          epochs=150, early_stopping=10,
                                                          filters_per_layer=[64, 128, 512],
                                                          layers_per_block=0, pool_every=4, hidden_dims=[],
                                                          checkpoints="./checkpoints/mlp_trained_full_with_interp_50_p",
                                                          lr=0.00001, features_loss_fns=features_loss_fns,
                                                          features_loss_weights=features_loss_weights,
                                                          label_loss_fns=label_loss_fns,
                                                          label_loss_weights=label_loss_weights,
                                                          model_type="cnn", batches=None,
                                                          train_nn_space=val_nn_space,
                                                          val_nn_space=val_nn_space,
                                                          test_nn_space=test_nn_space
                                                          )

    #
    # print("Training Pretrained model with interp only")
    # model = torch.load(
    #     r'/media/hovav/1df0a4b3-8e8b-4fe3-bd7d-499ea780a5a8/home/liorshai/CGMproject/FaceRecognition/checkpoints/mlp_pretrain_30_03_2022_19_39_39')
    #
    # fit_res_mlp = DeepLearning.experiments.cnn_experiment(model=model, run_name="rs50_features",
    #                                                       ds_train=train_set,
    #                                                       ds_val=val_set, ds_test=test_set,
    #                                                       bs_train=batch_size, bs_val=batch_size, optimizer=None,
    #                                                       epochs=150, early_stopping=10,
    #                                                       filters_per_layer=[64, 128, 512],
    #                                                       layers_per_block=0, pool_every=4, hidden_dims=[],
    #                                                       checkpoints="./checkpoints/mlp_emb_with_pretrained",
    #                                                       lr=0.001, features_loss_fns=features_loss_fns,
    #                                                       features_loss_weights=features_loss_weights,
    #                                                       label_loss_fns=label_loss_fns,
    #                                                       label_loss_weights=label_loss_weights,
    #                                                       model_type="cnn", batches=None,
    #                                                       train_nn_space=val_nn_space,
    #                                                       val_nn_space=val_nn_space,
    #                                                       test_nn_space=test_nn_space
    #                                                       )
    print("Finished")

if __name__ == "__main__":
    torch.backends.cudnn.benchmark = True
    parser = argparse.ArgumentParser(description='PyTorch ArcFace Training')
    parser.add_argument('config', type=str, help='py config file')
    parser.add_argument('--local_rank', type=int, default=0, help='local_rank')
    main(parser.parse_args())


class Point:
    def move(self):
        print("draw a line")

    def join(self):
        print("join the points")

point1 = Point()
point1.x=10
point1.y=20
point1.join()
point1.move()
print(point1.x)


# this datalogger exports the data considered as results for ECOS proceedings
from src.common.Datalogger import Datalogger
# from src.tools.GraphAndTex import __default_graph_options__

class MismatchDatalogger(Datalogger):  # a sub-class of dataloggers designed to export values concerning the whole run

    def __init__(self, period=1, graph_options="default"):
        if period == "global":
            super().__init__("mismatch_global", "Mismatch_global", period)
        else:
            super().__init__(f"mismatch_frequency_{period}", f"Mismatch_frequency_{period}", period, graph_options=graph_options, graph_labels={"xlabel": "time", "ylabel": f"mismatch"})

        if self._type != "global":
            self.add(f"simulation_time", graph_status="X")
            self.add(f"physical_time", graph_status="")

        natures_list = self._catalog.get("dictionaries")['natures'].keys()  # get all the names

        def create_energy_mismatch_function(nature_name):

            def get_energy_mismatch(name):
                energy_mismatch = self._catalog.get(f"{nature_name}.energy_consumed") - self._catalog.get(f"{nature_name}.energy_produced")

                return energy_mismatch

            return get_energy_mismatch

        for nature_name in natures_list:  # for each aggregator registered into world, all the relevant keys are added
            self.add(f"{nature_name}.energy_mismatch", create_energy_mismatch_function(nature_name))


import random

def dind_min_elem(dane):
    min = dane(0)
    for i in dane:
        if i<min:
            min = i
    return min

def find_poz_min_elem(dane):
    min_poz=0
    for i in range(1, n):
        if dane(i)<dane[min_poz]:
            min_poz=i
        return min_poz

L = []
for i in range(20):
    lista.append(random.randint(10, 50))
print(L)
print("Najmniejszy element listy to: ",find_min_elem(L))
print("Pozycja tego elementu to: ",find_poz_min_elem(L)+1)

print(find_poz_min_elem(L))
print("Najmniejszy element listy to: ",L[poz])
print("Pozycja tego elementu to: ",poz+1)


import numpy as np
from cffi import FFI

__all__ = [ 'subsample2' ]

ffi = FFI()
ffi.cdef('''

void downsample2(const double* x,             // shape: n x point_dim
                 const int* idx_selected,     // shape: k
                 int k,
                 const int* idx_not_selected, // shape: n-k
                 unsigned char* selected,     // shape: n
                 double* d_min,               // shape: n
                 int n,                       // number of points
                 int point_dim,
                 int n_pts_to_add);

''')

so = ffi.dlopen('subsample2.so')
downsample = so.downsample2

def const_double_ptr(arr):
    assert arr.dtype == np.float64, arr.dtype
    return ffi.cast('const double*', arr.ctypes.data)

def double_ptr(arr):
    assert arr.dtype == np.float64, arr.dtype
    return ffi.cast('double*', arr.ctypes.data)

def uchar_ptr(arr):
    assert arr.dtype == np.bool_, arr.dtype
    return ffi.cast('unsigned char*', arr.ctypes.data)

def const_int_ptr(arr):
    assert arr.dtype == np.intc, arr.dtype
    return ffi.cast('const int*', arr.ctypes.data)

#-------------------------------------------------------------------------------

def subsample2(x, n_desired_pts):
    # Returns: a boolean array, indicating whether a point in x was selected.
    assert x.ndim == 2, x.shape # x is a 2D array of the points
    n_pts, point_dim = x.shape[0], x.shape[1]
    assert n_pts > 0 and point_dim > 0, (n_pts, point_dim)
    assert np.isfinite(x).all()
    assert 0 < n_desired_pts, n_desired_pts
    if n_desired_pts > n_pts:
        msg = '*** Warning: We have less points than desired! ({} < {}) ***'
        print(msg.format(n_pts, n_desired_pts))
        return np.full(n_pts, np.True_, dtype=np.bool_)
    else:
        # Best effort: try to pick the point, closest to the middle:
        mean = np.mean(x, axis=0)
        idx = np.argmin(np.linalg.norm(x-mean, axis=1, ord=1))
        selected = np.full(n_pts, np.False_, dtype=np.bool_)
        selected[idx] = np.True_
        # Then call subsampling to fill up the rest, if any
        return subsample2_(x, selected, n_desired_pts)

def subsample2_(x, selected, n_desired_pts):
    # Similar to subsample but some points are already selected
    assert x.ndim == 2, x.shape # x is a 2D array of the points
    n_pts, point_dim = x.shape[0], x.shape[1]
    assert n_pts > 0 and point_dim > 0, (n_pts, point_dim)
    assert np.isfinite(x).all()
    assert x.shape[0] == selected.shape[0], (x.shape, selected.shape)
    assert 0 < n_desired_pts, n_desired_pts
    if n_desired_pts > n_pts:
        msg = '*** Warning: We have less points than desired! ({} < {}) ***'
        print(msg.format(n_pts, n_desired_pts))
        return np.full(n_pts, np.True_, dtype=np.bool_)
    selected = selected.copy()
    all_idx = np.arange(n_pts)
    idx = all_idx[selected]
    k = len(idx)
    to_add = n_desired_pts - k
    if to_add <= 0:
        return selected
    idx_not_selected = all_idx[~selected]
    assert len(idx_not_selected) == n_pts - k
    assert selected.any()
    x = np.ascontiguousarray(x)
    idx = idx.astype(np.intc)
    idx_not_selected = idx_not_selected.astype(np.intc)
    # d_nn: temporary work array for distances to the nearest neighbors
    d_nn = np.full(n_pts, np.nan)
    downsample(const_double_ptr(x), # shape: (n_pts, point_dim)
               const_int_ptr(idx),  # shape: (k,)
               k,
               const_int_ptr(idx_not_selected), # shape: (n_pts-k,)
               uchar_ptr(selected), # shape: (n_pts,)
               double_ptr(d_nn),    # shape: (n_pts,)
               n_pts,
               point_dim,
               to_add)
    return selected

#-------------------------------------------------------------------------------
# For performance testing and profiling with perf

def _main():
    np.random.seed(1)
    # 10k -> 1k downsampling, point dimension: 25
    subsample2(np.random.random((10000, 25)), 1000)

if __name__ == '__main__':
    _main()


import sys
input = sys.stdin.readline

words = []
N = int(input())
for _ in range(N):
    word = input().strip()  # 입력에서 줄바꿈 제거
    words.append(word)
words = list(set(words))  # 동일한 단어를 제거하기위해 set으로 한번 변환해준 뒤 다시 list로 변환

words.sort()
words.sort(key=lambda word : len(word))  # words내의 길이를 기준으로 정렬
for answer in words:
    print(answer)


# csv = comma separated values

""" import csv
with open('227 - weather-data.csv') as data:
    df = csv.reader(data)
    temp = []
    for row in df:
        if row[1] == 'temp':
            pass
        else:
            temp.append(int(row[1]))
    print(temp) """

import pandas as pd

df = pd.read_csv('227 - weather-data.csv')

#print(type(df)) # it gives type of dataframe which is 2 d
#print(type(df['temp'])) # it gives type of series which is 1 d
#print(df.to_dict())

temp_list = df['temp'].to_list()
avg = sum(temp_list) / len(temp_list)
#print(avg)

#print(df['temp'].mean())
#print(df['temp'].max())

# GEt data in coulmn
#print(df['temp'])
#print(df.temp)

# get data in row
#print(df[df.day =="Wednesday"])
#print(df[df.temp == df['temp'].max()])

monday = df[df.day == "Monday"]
monday_temp = int(monday.temp) * (9/5) + 32
print(monday_temp)

# create a dataframe from scratch

dict_22 = {
    "Students":["Amy","Abbas","ALi","Mohmaed"],
    "Scores":[12,43,45,16]
}

df22 = pd.DataFrame(dict_22)
print(df22)
df22.to_csv('New.csv')


# Given a directed, acyclic graph of N nodes.  Find all possible paths from node 0 to node N-1, and return them in any order.

# The graph is given as follows:  the nodes are 0, 1, ..., graph.length - 1.  graph[i] is a list of all nodes j for which the edge (i, j) exists.

import copy
class Solution(object):

    def findPath(self, graph, u, v, path, visited):
        visited[u] = True
        path.append(u)
        if u == v:
            output.append(copy.copy(path))

        else:
            for i in graph[u]:
                if not visited[i]:
                    self.findPath(graph,i,v,path,visited)

        visited [u] = False
        path.pop()

    def allPathsSourceTarget(self, graph):
        """
        :type graph: List[List[int]]
        :rtype: List[List[int]]
        """
        global output
        output = []
        visited = [False for i in range(len(graph))]
        path = []
        self.findPath(graph, 0, len(graph)-1, path, visited)
        return output


import datetime

from flask import g
from flask_restful import Resource, reqparse, abort, fields, marshal
from sqlalchemy.sql.elements import or_, and_

from App.apis.api_constant import HTTP_OK
from App.apis.movie_user_apis.utils import login_required, required_permmsion
from App.ext import db
from App.modeles.cinema_admin.cinema_hall_model import Hall
from App.modeles.cinema_admin.hall_movie_model import HallMovie
from App.modeles.user.model_constant import VIP_USER
from App.modeles.user.movie_order_model import MovieOrder, ORDER_STATUS_PAYED_NOT_GET, ORDER_STATUS_NOT_PAY, \
    ORDER_STATUS_GET

parse = reqparse.RequestParser()
parse.add_argument("hall_movie_id",required = True,help="请提供排挡信息")
parse.add_argument("o_seats",required=True,help="请选择座位号")

movie_order_fields = {
    "o_price":fields.Float,
    "o_seats":fields.String,
    "o_hall_movie_id":fields.Integer,
    "o_status":fields.String,
    "o_user_id":fields.Integer
}

class MovieUserOrderResource(Resource):

    @login_required
    def post(self):
        user = g.user
        args = parse.parse_args()
        hall_movie_id = args.get("hall_movie_id")
        o_seats = args.get("o_seats")

        """
        座位存在
        座位没有被锁单或者下单
        """
        #可用座位编号

        hall = Hall.query.get(HallMovie.query.get(hall_movie_id).h_id)

        movie_order_buyed = MovieOrder.query.filter(MovieOrder.o_hall_movie_id == hall_movie_id).filter(or_(MovieOrder.o_status == ORDER_STATUS_PAYED_NOT_GET,MovieOrder.o_status == ORDER_STATUS_GET)).all()

        movie_order_locked = MovieOrder.query.filter(MovieOrder.o_hall_movie_id == hall_movie_id).filter(and_(MovieOrder.o_status == ORDER_STATUS_NOT_PAY,MovieOrder.o_time > datetime.datetime.now())).all()

        print("movie_order_buyed",movie_order_buyed)
        print("movie_order_locked",movie_order_locked)

        seats_lock =[]

        for movie_order in movie_order_buyed:
            seats = movie_order.o_seats.split("#")

            seats_lock += seats

        for movie_order in movie_order_locked:
            seats = movie_order.o_seats.split("#")
            seats_lock += seats

        print("seats_lock",seats_lock)

        print("hall",hall)

        seats_can_buy = list(set(hall.h_seate.split("#")) ^ set(seats_lock))

        o_seats_list = o_seats.split("#")
        #判断当前选择的座位号是不是可选择座位号的子集
        if not set(o_seats_list).issubset(set(seats_can_buy)):
            abort("座位选择错误，请重新选择")
        #可用座位编号

        mover_order = MovieOrder()
        mover_order.o_hall_movie_id = hall_movie_id
        mover_order.o_seats = o_seats
        mover_order.o_user_id = user.id
        mover_order.o_time = datetime.datetime.now() + datetime.timedelta(minutes=15)

        '''
        try:
            movie_order1 = MovieOrder.query.get(1)
            movie_order2 = MovieOrder.query.get(2)
            db.session.delete(movie_order1)
            db.session.delete(movie_order2)
        except Exception as e:
            print(e)
            db.session.rollback()

        else:
            db.session.commit()
        '''

        if not mover_order.save():
            abort(400,msg="下单失败")

        data = {
            "msg":"下单成功",
            "status":HTTP_OK,
            "data":marshal(mover_order,movie_order_fields)
        }
        return data

class MovieUserOrdersResource(Resource):

    @required_permmsion(VIP_USER)
    def put(self,orderid):

        return {"msg":"oook"}


import socket
import time
import controller

def init():
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    # 建立连接:
    s.connect(('106.14.149.194', 9990))
    s.setblocking(True)
    s.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1024)
    s.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 50240)
    return s

def resetNet(type, s=None):
    while True:
        try:
            if s != None:
                s.close()
            s = init()
            s.send(type.to_bytes(4, byteorder="big"))
        except BaseException:
            time.sleep(2)
            resetNet(type, s)
        else:
            break
    return s

def waitCommand():
    s1 = resetNet(10000)
    while True:
        try:
            cmd = s1.recv(4)
        except BaseException:
            s1 = resetNet(10000, s1)
            continue
        cmd = int.from_bytes(cmd, byteorder="big")
        controller.action(cmd - 1)

def start():
    waitCommand()