import copy
import json
import os
import re
import shutil
from collections import defaultdict
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional, TextIO, Type, Union

import click
import htmlmin
import jinja2
import markdown2
import yaml
from jinja2 import FileSystemLoader
from pygments import highlight
from pygments.formatters.html import HtmlFormatter
from pygments.lexers.javascript import JavascriptLexer
from pytz import reference

TEMPLATE_FOLDER = "templates"
TEMPLATE_FILE_NAME = "base.html"
CSS_FILE_NAME = "schema_doc.css"
JS_FILE_NAME = "schema_doc.min.js"

DEFAULT_PATTERN = r"(\[Default - `([^`]+)`\])"
DEPRECATED_PATTERN = r"\[Deprecated"

TYPE_ARRAY = "array"
TYPE_BOOLEAN = "boolean"
TYPE_CONST = "const"
TYPE_ENUM = "enum"
TYPE_INTEGER = "integer"
TYPE_NUMBER = "number"
TYPE_OBJECT = "object"
TYPE_STRING = "string"

DESCRIPTION = "description"
DEFAULT = "default"
EXAMPLES = "examples"
ITEMS = "items"
TYPE = "type"
REF = "$ref"

MULTIPLE_OF = "multipleOf"
MAXIMUM = "maximum"
EXCLUSIVE_MAXIMUM = "exclusiveMaximum"
MINIMUM = "minimum"
EXCLUSIVE_MINIMUM = "exclusiveMinimum"

SHORT_DESCRIPTION_NUMBER_OF_LINES = 8

class SchemaNode:
    """
    Represents a part of a JSON schema with additional metadata to help with documentation
    """

    def __init__(
        self,
        depth: int,
        file: str,
        path_to_element: List[Union[str, int]],
        html_id: str,
        literal: Union[str, int] = None,
        keywords: Dict[str, "SchemaNode"] = None,
        array_items: List["SchemaNode"] = None,
        refers_to: "SchemaNode" = None,
        is_displayed: bool = True,
    ):
        """

        :param depth: Number of levels from the root of the schema to this node.
        :param file: Real path to the schema file
        :param path_to_element: Path from the root of the schema to the current element
        :param html_id: HTML ID for the current element. Used for anchor links.
        :param literal: If the schema is neither a dict nor an array, it will be kept here
                        Useful for things like description, types, const, enum, etc.
        :param keywords: If the schema is a dict, this will be filled. Otherwise, this stays empty
        :param array_items: If the schema is an array, this will be filled. Otherwise, this stays empty
        :param refers_to: If there is a $ref, this should contain the SchemaNode object for it
        :param is_displayed: Instructs the templates if this part should be fully documented.
                             If false, the description and a link to the referenced element will be generated instead.
                             If false, refers_to needs to be set
        """
        self.depth = depth
        self.file = file
        self.path_to_element = path_to_element
        self.literal = literal
        self.keywords = keywords or {}
        self.array_items = array_items or []
        self.refers_to = refers_to
        self.is_displayed = is_displayed
        self.html_id = html_id or "_".join(path_to_element) or "root"

    @property
    def is_definition(self) -> bool:
        return self.path_to_element and self.path_to_element[0] != "definitions"

    def __eq__(self, other: object) -> bool:
        """For two schema nodes to be considered equals they must represent the same element in the same file"""
        if other is None:
            return False

        if not isinstance(other, SchemaNode):
            return NotImplemented

        return self.file == other.file and self.path_to_element == other.path_to_element

def build_intermediate_representation(
    schema_path: Union[str, TextIO], loaded_schemas: Optional[Dict[str, Any]] = None
) -> SchemaNode:
    """Build a SchemaNode object representing a JSON schema with added metadata to help rendering as a documentation.

    The representation will resolve references and generate HTML ids for elements
    """
    resolved_references: Dict[str, Dict[str, SchemaNode]] = defaultdict(dict)

    def defaultdict_list() -> Dict[Any, List]:
        return defaultdict(list)

    reference_users: Dict[str, Dict[str, List[SchemaNode]]] = defaultdict(defaultdict_list)
    _loaded_schemas: Dict[str, Any]
    if loaded_schemas is None:
        _loaded_schemas = {}
    else:
        assert isinstance(loaded_schemas, dict) and all(
            isinstance(k, str) for k in loaded_schemas.keys()
        ), "loaded_schemas must be Dict[str, Any]"
        _loaded_schemas = loaded_schemas

    # Make sure schema_path is absolute, all symlinks are resolved
    if isinstance(schema_path, Path):
        schema_path = str(schema_path.resolve())
    elif isinstance(schema_path, str):
        schema_path = os.path.realpath(schema_path)
    else:
        # Assuming schema_path is a file object (TextIO)
        schema_path = os.path.realpath(schema_path.name)

    def _record_ref(schema_real_path: str, path_to_element: List[Union[str, int]], current_node: SchemaNode) -> None:
        """Record that the node is describing the schema at the provided path"""
        resolved_references[schema_real_path]["/".join(str(e) for e in path_to_element)] = current_node

    def _resolve_ref(current_node: SchemaNode, schema: Union[Dict, List, int, str]) -> Optional[SchemaNode]:
        """Resolve the $ref keyword

        If there is no $ref, return None.
        If there is a referenced element that was never encountered before, build that element and return it.
        If there is a referenced element that was already encountered, return it.

        This method also makes sure that the final element to be fully documented is the one that is the less nested
        so that the information is closer to the user.
        """
        if not isinstance(schema, Dict) or REF not in schema:
            return None

        reference_path = schema.get(REF)
        if not reference_path:
            return None

        # Reference found, resolve the path (format "#/a/b/c", "file.json#/a/b/c", or "file.json")
        if "#" not in reference_path:
            file_path_part = reference_path
            anchor_part = ""
        else:
            file_path_part, anchor_part = reference_path.split("#", maxsplit=1)
            anchor_part = anchor_part.strip("/")

        # Resolve file path portion of reference
        if file_path_part:
            referenced_schema_path = os.path.realpath(os.path.join(os.path.dirname(current_node.file), file_path_part))
        else:
            referenced_schema_path = os.path.realpath(current_node.file)

        def _find_reference(path: str, anchor_path: str) -> Optional[SchemaNode]:
            resolved_references_for_this_schema = resolved_references[path]
            return resolved_references_for_this_schema.get(anchor_path)

        # Check if already loaded
        found_reference = _find_reference(referenced_schema_path, anchor_part)

        if found_reference == current_node:
            found_reference = None

        if found_reference:
            while not found_reference.is_displayed and found_reference.refers_to:
                if found_reference.refers_to == current_node:
                    break
                found_reference = found_reference.refers_to

            # Is someone else using the reference?
            reference_users_for_this_schema = reference_users[found_reference.file][anchor_part]
            reference_users[referenced_schema_path][anchor_part].append(current_node)
            if reference_users_for_this_schema:
                other_user = None
                other_is_better = False
                i_am_better = False
                for user in reference_users_for_this_schema:
                    if user == current_node or not user.is_displayed:
                        continue

                    if not other_user:
                        other_user = user

                    if user.depth < other_user.depth:
                        other_user = user

                    if other_user.depth < current_node.depth:
                        other_user = user
                        other_is_better = True
                        i_am_better = False
                    elif other_user.depth > current_node.depth:
                        other_is_better = False
                        i_am_better = True

                # There is at least on other node having the same reference as the current node.
                if other_is_better:
                    # The other referencing node is nearer to the user, so it will now be displayed
                    # We mark the current node as being hidden and linking to the other one
                    other_user.is_displayed = True
                    current_node.is_displayed = False
                    return other_user
                elif i_am_better:
                    # The other referencing node is more nested, it should be hidden and link to the current node
                    # The current node will documented the element referenced by both
                    other_user.is_displayed = False
                    other_user.refers_to = current_node
                    current_node.is_displayed = True
                    return found_reference
                else:
                    # Both nodes are the same depth. The other having been seen first,
                    # this node will be hidden and link to it
                    current_node.is_displayed = False
                    return other_user

            return found_reference
        else:
            reference_users[referenced_schema_path][anchor_part].append(current_node)

        # Not an existing reference, so it shall be built
        referenced_schema_path_to_element = anchor_part.split("/")
        return _build_node(
            current_node.depth,
            current_node.html_id,
            referenced_schema_path,
            referenced_schema_path_to_element,
            _load_schema(referenced_schema_path, referenced_schema_path_to_element),
        )

    def _load_schema(schema_file_path: str, path_to_element: List[Union[str, int]]) -> Union[Dict, List, int, str]:
        """Load the schema at the provided path. The path must be a "realpath", meaning absolute and with symlinks
        resolved.

        Loaded paths are kept in memory as to ensure never loading the same file twice
        """
        if schema_file_path in _loaded_schemas:
            loaded_schema = _loaded_schemas[schema_file_path]
        else:
            with open(schema_file_path, encoding="utf-8") as schema_fp:
                _, extension = os.path.splitext(schema_file_path)
                if extension == ".json":
                    loaded_schema = json.load(schema_fp)
                else:
                    loaded_schema = yaml.safe_load(schema_fp)
            _loaded_schemas[schema_file_path] = loaded_schema

        if path_to_element:
            for path_part in path_to_element:
                if not path_part:
                    # Empty string
                    continue
                if isinstance(path_part, str):
                    loaded_schema = loaded_schema[path_part]
                elif isinstance(path_part, int):
                    loaded_schema = loaded_schema[path_part]

        return loaded_schema

    def _build_node(
        depth: int,
        html_id: str,
        schema_file_path: str,
        path_to_element: List[Union[str, int]],
        schema: Union[Dict, List, int, str],
        is_pattern_properties: bool = False,
    ) -> SchemaNode:
        """Recursively build a schema representation

        :param depth: Number of levels from the root of the schema to this node. Used when there are references to
                      figure out the less nested one in order to display it.
        :param html_id: HTML ID for the current element. Used for anchor links.
        :param schema_file_path: Real path to the schema (absolute path with symlinks resolved)
        :param path_to_element: Path from the root of the schema to the current element
        :param schema: The JSON schema part being represented
        :param is_pattern_properties: True if this node is under the keyword "patternProperties"
        :return: A representation of the schema
        """
        schema_file_path = os.path.realpath(schema_file_path)

        new_node = SchemaNode(depth, file=schema_file_path, path_to_element=path_to_element, html_id=html_id)
        if html_id == "root":
            html_id = ""

        _record_ref(schema_file_path, path_to_element, new_node)

        if isinstance(schema, dict):
            keywords = {}
            pattern_id = 1
            for schema_key, schema_value in schema.items():
                # These won't be needed to render the documentation.
                # The definitions will be reached from references, otherwise they are useless
                if schema_key in ["$id", "$ref", "$schema", "definitions"]:
                    continue

                # Examples are rendered in JSON because they will be represented that way in the documentation,
                # no need for a SchemaNode object
                if schema_key == "examples":
                    keywords[schema_key] = [
                        json.dumps(example, indent=4, separators=(",", ": "), ensure_ascii=False)
                        for example in schema_value
                    ]
                    continue

                # The default value will be printed as-is, no need for a SchemaNode object
                if schema_key == "default":
                    keywords[schema_key] = json.dumps(schema_value, ensure_ascii=False)
                    continue

                # Add the property name (correctly escaped) to the ID
                new_html_id = html_id
                new_depth = depth
                if schema_key not in ["properties", "patternProperties"]:
                    new_depth += 1
                    new_html_id += "_" if html_id else ""
                    if not is_pattern_properties:
                        new_html_id += escape_property_name_for_id(schema_key)
                    else:
                        new_html_id += f"pattern{pattern_id}"
                        pattern_id += 1

                property_path = copy.deepcopy(path_to_element) + [schema_key]
                keywords[schema_key] = _build_node(
                    new_depth,
                    new_html_id,
                    schema_file_path,
                    property_path,
                    schema_value,
                    is_pattern_properties=schema_key == "patternProperties",
                )
            new_node.keywords = keywords
        elif isinstance(schema, list):
            array_items = []
            for i, element in enumerate(schema):
                # Add the property name (correctly escaped) to the ID
                new_html_id = html_id + ("_" if html_id else "") + "i" + str(i)

                array_items.append(
                    _build_node(depth + 1, new_html_id, schema_file_path, path_to_element + [i], element)
                )
            new_node.array_items = array_items

        else:
            new_node.literal = schema

        new_node.refers_to = _resolve_ref(new_node, schema)

        return new_node

    intermediate_representation = _build_node(0, "", schema_path, [], _load_schema(schema_path, []))

    return intermediate_representation

def is_combining(schema_node: SchemaNode) -> bool:
    """Test if a schema is one of the combining schema keyword"""
    return bool({"anyOf", "allOf", "oneOf", "not"}.intersection(schema_node.keywords.keys()))

def is_text_short(text: str) -> bool:
    """Check if a string is short so that we can decide whether to make the section containing it expandable or not.
    The heuristic is counting 1 for each line + 1 for each group of 80 characters a line has
    """
    return sum((len(line) / 80 + 1) for line in str(text).splitlines()) < SHORT_DESCRIPTION_NUMBER_OF_LINES

def is_deprecated(property_dict: Dict[str, Any]) -> bool:
    """Test. Check if a property is deprecated without looking in description"""
    return False

def is_deprecated_look_in_description(schema_node: SchemaNode) -> bool:
    """Test. Check if a property is deprecated looking in description"""
    if DESCRIPTION not in schema_node.keywords:
        return False

    return bool(re.match(DEPRECATED_PATTERN, schema_node.keywords[DESCRIPTION].literal))

def get_required_properties(schema_node: SchemaNode) -> List[str]:
    required_properties = schema_node.keywords.get("required") or []
    if required_properties:
        required_properties = [p.literal for p in required_properties.array_items]

    return required_properties

def get_undocumented_required_properties(schema_node: SchemaNode) -> List[str]:
    required_properties = get_required_properties(schema_node)

    documented_properties = schema_node.keywords.get("properties")
    documented_properties = documented_properties.keywords.keys() if documented_properties else []

    undocumented = []
    for property_name in required_properties:
        if property_name not in documented_properties:
            undocumented.append(property_name)

    return undocumented

def python_to_json(value: Any) -> Any:
    """Filter. Return the value as it needs to be displayed in JSON

    Used to display a string literals more explicitly for default and const values.
    """
    if value is None:
        return "null"
    if value is True:
        return "true"
    if value is False:
        return "false"

    if isinstance(value, str) and not value.startswith('"'):
        return f'"{value}"'

    return value

def get_type_name(schema_node: SchemaNode) -> str:
    """Filter. Return the type of a property taking into account the type of items for array and enum"""

    def _python_type_to_json_type(python_type: Type[Union[str, int, float, bool, list, dict]]) -> str:
        return {
            str: TYPE_STRING,
            int: TYPE_INTEGER,
            float: TYPE_NUMBER,
            bool: TYPE_BOOLEAN,
            list: TYPE_ARRAY,
            dict: TYPE_OBJECT,
        }.get(python_type, TYPE_STRING)

    def _enum_type(enum_values: List[SchemaNode]) -> str:
        enum_type_names = [
            _python_type_to_json_type(python_type_name)
            for python_type_name in set(type(v.literal) for v in enum_values)
        ]
        if enum_type_names:
            return f"{TYPE_ENUM} (of {' or '.join(enum_type_names)})"

        return TYPE_ENUM

    def _add_subtype_if_array(type_name: str):
        if type_name == TYPE_ARRAY:
            items = schema_node.keywords.get(ITEMS, None)
            if not items:
                return type_name

            subtype = items.keywords.get(TYPE)
            if subtype:
                subtype = subtype.literal
            if TYPE_ENUM in items.keywords:
                subtype = _enum_type(items.keywords[TYPE_ENUM].array_items)

            if not subtype:
                # Too complex to guess items
                return type_name

            type_name = f"{type_name} of {subtype}"

        return type_name

    if TYPE_CONST in schema_node.keywords:
        return TYPE_CONST
    if TYPE_ENUM in schema_node.keywords:
        return _enum_type(schema_node.keywords[TYPE_ENUM].array_items)

    type_node = schema_node.keywords.get(TYPE)
    if type_node:
        if type_node.array_items:
            type_names = [node.literal for node in type_node.array_items]
        else:
            type_names = [type_node.literal]
    else:
        type_names = [TYPE_OBJECT]

    type_names = [_add_subtype_if_array(type_name) for type_name in type_names]

    return ", ".join(type_names[:-1]) + (" or " if len(type_names) > 1 else "") + type_names[-1]

def _get_description(schema_node: SchemaNode) -> str:
    description = ""
    description_node = schema_node.keywords.get(DESCRIPTION)
    if description_node:
        description = description_node.literal

    seen = set()
    current_node = schema_node
    while not description and current_node.refers_to:
        if current_node.html_id in seen:
            break
        seen.add(current_node.html_id)
        referenced_schema = current_node.refers_to
        referenced_description_node = referenced_schema.keywords.get(DESCRIPTION)
        if referenced_description_node:
            description = referenced_description_node.literal
        current_node = referenced_schema

    return description

def get_description(schema_node: SchemaNode) -> str:
    """Filter. Get the description of a property or an empty string"""
    return _get_description(schema_node)

def get_description_remove_default(schema_node: SchemaNode) -> str:
    """Filter. From the description attribute of a property, return the description without any default values in it.
    Will also convert None to an empty string.
    """
    description = _get_description(schema_node)
    if not description:
        return ""

    match = re.match(DEFAULT_PATTERN, description)
    if not match:
        return description

    return description[match.span(1)[1] :].lstrip()

def get_default(schema_node: SchemaNode) -> str:
    """Filter. Return the default value for a property"""
    return schema_node.keywords.get(DEFAULT) or ""

def get_default_look_in_description(schema_node: SchemaNode) -> str:
    """Filter. Get the default value of a JSON Schema property. If not set, look for it in the description."""
    default_value = get_default(schema_node)
    if default_value:
        return default_value

    description = schema_node.keywords.get(DESCRIPTION).literal
    if not description:
        return ""

    match = re.match(DEFAULT_PATTERN, description)
    if not match:
        return ""

    return match.group(2)

def get_numeric_restrictions_text(schema_node: SchemaNode, before_value: str = "", after_value: str = "") -> str:
    """Filter. Get the text to display about restrictions on a numeric type(integer or number)"""
    multiple_of = schema_node.keywords.get(MULTIPLE_OF)
    if multiple_of:
        multiple_of = multiple_of.literal
    maximum = schema_node.keywords.get(MAXIMUM)
    if maximum:
        maximum = maximum.literal
    exclusive_maximum = schema_node.keywords.get(EXCLUSIVE_MAXIMUM)
    if exclusive_maximum:
        exclusive_maximum = exclusive_maximum.literal
    minimum = schema_node.keywords.get(MINIMUM)
    if minimum:
        minimum = minimum.literal
    exclusive_minimum = schema_node.keywords.get(EXCLUSIVE_MINIMUM)
    if exclusive_minimum:
        exclusive_minimum = exclusive_minimum.literal

    # Fix minimum and exclusive_minimum both there
    if minimum is not None and exclusive_minimum is not None:
        if minimum <= exclusive_minimum:
            exclusive_minimum = None
        else:
            minimum = None

    minimum_fragment = ""
    if minimum is not None:
        minimum_fragment += f"greater or equal to {before_value}{minimum}{after_value}"
    if exclusive_minimum is not None:
        minimum_fragment += f"strictly greater than {before_value}{exclusive_minimum}{after_value}"

    # Fix maximum and exclusive_maximum both there
    if maximum is not None and exclusive_maximum is not None:
        if maximum > exclusive_maximum:
            exclusive_maximum = None
        else:
            maximum = None

    maximum_fragment = ""
    if maximum is not None:
        maximum_fragment += f"lesser or equal to {before_value}{maximum}{after_value}"
    if exclusive_maximum is not None:
        maximum_fragment += f"strictly lesser than {before_value}{exclusive_maximum}{after_value}"

    result = "Value must be "
    touched = False
    if minimum_fragment:
        touched = True
        result += minimum_fragment
    if maximum_fragment:
        if touched:
            result += " and "
        touched = True
        result += maximum_fragment
    if multiple_of:
        if touched:
            result += " and "
        result += f"a multiple of {before_value}{multiple_of}{after_value}"

    return result if touched else ""

def escape_property_name_for_id(property_name: str) -> str:
    """Filter. Escape unsafe characters in a property name so that it can be used in a HTML id"""

    escaped = re.sub("[^0-9a-zA-Z_,.-]", "_", str(property_name))
    if not escaped[0].isalpha():
        escaped = "a" + escaped
    return escaped

def highlight_json_example(example_text: str) -> str:
    """Filter. Return an highlighted version of the provided JSON text"""
    return highlight(example_text, JavascriptLexer(), HtmlFormatter())

def get_local_time() -> str:
    return datetime.now(tz=reference.LocalTimezone()).strftime("%Y-%m-%d at %H:%M:%S %z")

def generate_from_schema(
    schema_file: Union[str, Path, TextIO],
    loaded_schemas: Optional[Dict[str, Any]] = None,
    minify: bool = False,
    deprecated_from_description: bool = False,
    default_from_description: bool = False,
    expand_buttons: bool = False,
    link_to_reused_ref: bool = True,
) -> str:

    template_folder = os.path.join(os.path.dirname(__file__), TEMPLATE_FOLDER)
    base_template_path = os.path.join(template_folder, TEMPLATE_FILE_NAME)

    md = markdown2.Markdown(extras={"fenced-code-blocks": {"cssclass": "highlight jumbotron"}})
    loader = FileSystemLoader(template_folder)
    env = jinja2.Environment(loader=loader)
    env.filters["markdown"] = lambda text: jinja2.Markup(md.convert(text))
    env.filters["python_to_json"] = python_to_json
    env.filters["get_default"] = get_default_look_in_description if default_from_description else get_default
    env.filters["get_type_name"] = get_type_name
    env.filters["get_description"] = get_description_remove_default if default_from_description else get_description
    env.filters["get_numeric_restrictions_text"] = get_numeric_restrictions_text
    env.filters["get_required_properties"] = get_required_properties
    env.filters["get_undocumented_required_properties"] = get_undocumented_required_properties
    env.filters["highlight_json_example"] = highlight_json_example
    env.tests["combining"] = is_combining
    env.tests["description_short"] = is_text_short
    env.tests["deprecated"] = is_deprecated_look_in_description if deprecated_from_description else is_deprecated
    env.globals["get_local_time"] = get_local_time

    with open(base_template_path, "r") as template_fp:
        template = env.from_string(template_fp.read())

    if isinstance(schema_file, list):
        # Backward compatibility
        schema_file = os.path.sep.join(schema_file)

    intermediate_schema = build_intermediate_representation(schema_file, loaded_schemas)

    rendered = template.render(
        schema=intermediate_schema, expand_buttons=expand_buttons, link_to_reused_ref=link_to_reused_ref
    )

    if minify:
        rendered = htmlmin.minify(rendered)

    return rendered

def generate_from_filename(
    schema_file_name: Union[str, Path],
    result_file_name: str,
    minify: bool = True,
    deprecated_from_description: bool = False,
    default_from_description: bool = False,
    expand_buttons: bool = False,
    copy_css: bool = True,
    copy_js: bool = True,
    link_to_reused_ref: bool = True,
) -> None:
    if isinstance(schema_file_name, str):
        schema_file_name = os.path.realpath(schema_file_name)
    elif isinstance(schema_file_name, Path):
        schema_file_name = str(schema_file_name.resolve())

    rendered_schema_doc = generate_from_schema(
        schema_file_name,
        minify=minify,
        deprecated_from_description=deprecated_from_description,
        default_from_description=default_from_description,
        expand_buttons=expand_buttons,
        link_to_reused_ref=link_to_reused_ref,
    )

    copy_css_and_js_to_target(result_file_name, copy_css, copy_js)

    with open(result_file_name, "w", encoding="utf-8") as result_schema_doc:
        result_schema_doc.write(rendered_schema_doc)

def generate_from_file_object(
    schema_file: TextIO,
    result_file: TextIO,
    minify: bool,
    deprecated_from_description: bool,
    default_from_description: bool,
    expand_buttons: bool,
    copy_css: bool = True,
    copy_js: bool = True,
    link_to_reused_ref: bool = True,
) -> None:
    """Generate the JSON schema documentation from opened file objects for both input and output files. The
    result_file should be opened in write mode.
    """
    result = generate_from_schema(
        schema_file,
        minify=minify,
        deprecated_from_description=deprecated_from_description,
        default_from_description=default_from_description,
        expand_buttons=expand_buttons,
        link_to_reused_ref=link_to_reused_ref,
    )

    copy_css_and_js_to_target(result_file.name, copy_css, copy_js)

    result_file.write(result)

def copy_css_and_js_to_target(result_file_path: str, copy_css: bool, copy_js: bool) -> None:
    """Copy the CSS and JS files needed to display the resulting page to the directory containing the result file"""
    files_to_copy = []
    if copy_css:
        files_to_copy.append(CSS_FILE_NAME)
    if copy_js:
        files_to_copy.append(JS_FILE_NAME)
    if not files_to_copy:
        return

    target_directory = os.path.dirname(result_file_path)
    source_directory = os.path.dirname(__file__)
    if target_directory == source_directory:
        return

    for file_to_copy in files_to_copy:
        try:
            shutil.copy(os.path.join(source_directory, file_to_copy), os.path.join(target_directory, file_to_copy))
        except shutil.SameFileError:
            print(f"Not copying {file_to_copy} to {os.path.abspath(target_directory)}, file already exists")

@click.command()
@click.argument("schema_file", nargs=1, type=click.File("r", encoding="utf-8"))
@click.argument("result_file", nargs=1, type=click.File("w+", encoding="utf-8"), default="schema_doc.html")
@click.option("--minify/--no-minify", default=True, help="Run minification om the HTML result")
@click.option(
    "--deprecated-from-description", is_flag=True, help="Look in the description to find if an attribute is deprecated"
)
@click.option(
    "--default-from-description", is_flag=True, help="Look in the description to find an attribute default value"
)
@click.option("--expand-buttons", is_flag=True, help="Add 'Expand all' and 'Collapse all' buttons at the top")
@click.option("--copy-css/--no-copy-css", default=True, help=f"Copy {CSS_FILE_NAME} to the folder of the result_file")
@click.option("--copy-js/--no-copy-js", default=True, help=f"Copy {JS_FILE_NAME} to the folder of the result_file")
@click.option(
    "--link-to-reused-ref/--no-link-to-reused-ref",
    default=True,
    help="If set and 2 parts of the schema refer to the same definition, the definition will only be rendered once "
    "and all other references will be replaced by a link.",
)
def main(
    schema_file: TextIO,
    result_file: TextIO,
    minify: bool,
    deprecated_from_description: bool,
    default_from_description: bool,
    expand_buttons: bool,
    copy_css: bool,
    copy_js: bool,
    link_to_reused_ref: bool,
) -> None:
    generate_from_file_object(
        schema_file,
        result_file,
        minify,
        deprecated_from_description,
        default_from_description,
        expand_buttons,
        copy_css,
        copy_js,
        link_to_reused_ref,
    )

if __name__ == "__main__":
    main()


import math
import torch
from torch import nn
from torch.nn import functional as F
from torch.utils.checkpoint import checkpoint
from models.clinical_bert.model import ClinicalBertWrapper, EncoderSentences
from models.clusterer.model import Clusterer
from utils import traceback_attention as ta, entropy, set_dropout, set_require_grad, get_code_counts, tensor_to_none

class Model(nn.Module):
    def __init__(self, outdim=64, sentences_per_checkpoint=10, device1='cpu', device2='cpu', freeze_bert=True, code_embedding_type_params=set([]), concatenate_code_embedding=False, dropout=.15, cluster=False):
        super(Model, self).__init__()
        self.clinical_bert_sentences = EncoderSentences(ClinicalBertWrapper, embedding_dim=outdim, truncate_tokens=50, truncate_sentences=1000, sentences_per_checkpoint=sentences_per_checkpoint, device=device1)
        if freeze_bert:
            self.freeze_bert()
        else:
            self.unfreeze_bert(dropout=dropout)
        self.code_embedding_type_params = code_embedding_type_params
        num_code_embedding_types = len(code_embedding_type_params)
        self.code_embeddings = nn.Embedding(code_embedding_type_params['codes'][0], outdim)\
                               if 'codes' in code_embedding_type_params.keys() else None
        self.linearized_code_transformer = EncoderSentences(lambda : LinearizedCodesTransformer(num_linearization_embeddings), embedding_dim=outdim, truncate_tokens=50,
                                                            truncate_sentences=1000, sentences_per_checkpoint=sentences_per_checkpoint, device=device2)\
                                           if 'linearized_codes' in code_embedding_type_params.keys() else None
        self.attention = nn.MultiheadAttention(outdim, 1)
        self.concatenate_code_embedding = concatenate_code_embedding
        self.linear = nn.Linear(outdim, 1)
        self.linear2 = nn.Linear(outdim*num_code_embedding_types, outdim) if num_code_embedding_types > 1 else None
        self.linear3 = nn.Linear(2*outdim, outdim) if concatenate_code_embedding else None
        self.device1 = device1
        self.device2 = device2
        self.cluster = cluster
        self.clusterer = Clusterer() if cluster else None

    def freeze_bert(self):
        set_dropout(self.clinical_bert_sentences, 0)
        set_require_grad(self.clinical_bert_sentences, False)

    def unfreeze_bert(self, dropout=.15):
        set_dropout(self.clinical_bert_sentences, dropout)
        set_require_grad(self.clinical_bert_sentences, True)

    def correct_devices(self):
        self.clinical_bert_sentences.correct_devices()
        if self.code_embeddings is not None:
            self.code_embeddings.to(self.device2)
        self.attention.to(self.device2)
        self.linear.to(self.device2)
        if self.linear2 is not None:
            self.linear2.to(self.device2)
        self.codes_per_checkpoint = 1000
        if self.cluster:
            self.clusterer.to(self.device2)
        if self.concatenate_code_embedding:
            self.linear3.to(self.device2)

    def forward(self, article_sentences, article_sentences_lengths, num_codes, codes=None, code_description=None, code_description_length=None, linearized_codes=None, linearized_codes_lengths=None, linearized_descriptions=None, linearized_descriptions_lengths=None):
        nq = num_codes.max()
        nq_temp =  self.codes_per_checkpoint
        scores, attention, traceback_attention, context_vec = [], [], [], []
        for offset in range(0, nq, nq_temp):
            if codes is not None:
                codes_temp = codes[:, offset:offset+nq_temp]
            else:
                codes_temp = torch.zeros(0)
            if code_description is not None:
                code_description_temp = code_description[:, offset:offset+nq_temp]
                code_description_length_temp = code_description_length[:, offset:offset+nq_temp]
            else:
                code_description_temp = torch.zeros(0)
                code_description_length_temp = torch.zeros(0)
            if linearized_codes is not None:
                linearized_codes_temp = linearized_codes[:, offset:offset+nq_temp]
                linearized_codes_lengths_temp = linearized_codes_lengths[:, offset:offset+nq_temp]
            else:
                linearized_codes_temp = torch.zeros(0)
                linearized_codes_lengths_temp = torch.zeros(0)
            if linearized_descriptions is not None:
                linearized_descriptions_temp = linearized_descriptions[:, offset:offset+nq_temp]
                linearized_descriptions_lengths_temp = linearized_descriptions_lengths[:, offset:offset+nq_temp]
            else:
                linearized_descriptions_temp = torch.zeros(0)
                linearized_descriptions_lengths_temp = torch.zeros(0)
            num_codes_temp = torch.clamp(num_codes-offset, 0, nq_temp)
            scores_temp, attention_temp, traceback_attention_temp, context_vec_temp = checkpoint(
                self.inner_forward,
                article_sentences,
                article_sentences_lengths,
                num_codes_temp,
                codes_temp,
                code_description_temp,
                code_description_length_temp,
                linearized_codes_temp,
                linearized_codes_lengths_temp,
                linearized_descriptions_temp,
                linearized_descriptions_lengths_temp,
                *self.parameters())
            scores.append(scores_temp)
            attention.append(attention_temp)
            traceback_attention.append(traceback_attention_temp)
            context_vec.append(context_vec_temp)
        scores = torch.cat(scores, 1)
        attention = torch.cat(attention, 1)
        traceback_attention = torch.cat(traceback_attention, 1)
        context_vec = torch.cat(context_vec, 1)
        if self.cluster:
            clustering = self.clusterer(article_sentences, article_sentences_lengths, attention, num_codes)
        else:
            clustering = None
        return_dict = dict(
            scores=scores,
            num_codes=num_codes,
            attention=attention,
            traceback_attention=traceback_attention,
            article_sentences_lengths=article_sentences_lengths,
            clustering=clustering,
            context_vec=context_vec)
        if codes is not None:
            return_dict['codes'] = codes
        return return_dict

    def inner_forward(self, article_sentences, article_sentences_lengths, num_codes, codes, code_description, code_description_length, linearized_codes, linearized_codes_lengths, linearized_descriptions, linearized_descriptions_lengths, *args):
        codes, code_description, code_description_length, linearized_codes, linearized_codes_lengths, linearized_descriptions, linearized_descriptions_lengths = tensor_to_none(codes), tensor_to_none(code_description), tensor_to_none(code_description_length), tensor_to_none(linearized_codes), tensor_to_none(linearized_codes_lengths), tensor_to_none(linearized_descriptions), tensor_to_none(linearized_descriptions_lengths)
        encodings, self_attentions, word_level_attentions = self.clinical_bert_sentences(
            article_sentences, article_sentences_lengths)
        article_sentences_lengths, num_codes, encodings, self_attentions, word_level_attentions =\
            article_sentences_lengths.to(self.device2), num_codes.to(self.device2), encodings.to(self.device2), self_attentions.to(self.device2), word_level_attentions.to(self.device2)
        # b, ns, nt = word_level_attentions.shape
        b, ns, nl, nh, nt, _ = self_attentions.shape
        traceback_word_level_attentions = ta(
            self_attentions.mean(3).view(b*ns, nl, nt, nt),
            attention_vecs=word_level_attentions.view(b*ns, 1, nt))\
            .view(b, ns, nt)
        if codes is None and code_description is None and linearized_codes is None and linearized_descriptions is None: raise Exception
        all_code_embeddings = []
        if codes is not None:
            codes = codes.to(self.device2)
            all_code_embeddings.append(self.code_embeddings(codes))
        if code_description is not None:
            all_code_embeddings.append(self.clinical_bert_sentences(
                code_description,
                code_description_length,
            )[0].to(self.device2))
        if linearized_codes is not None:
            all_code_embeddings.append(self.linearized_code_transformer(
                linearized_codes,
                linearized_codes_lengths
            )[0])
        if linearized_descriptions is not None:
            all_code_embeddings.append(self.clinical_bert_sentences(
                linearized_descriptions,
                linearized_descriptions_lengths,
            )[0].to(self.device2))
        if self.linear2 is not None:
            code_embeddings = torch.cat(all_code_embeddings, 2)
            code_embeddings = self.linear2(code_embeddings)
        else:
            code_embeddings = all_code_embeddings[0]
        key_padding_mask = (article_sentences_lengths == 0)[:,:encodings.size(1)]
        contextvec, sentence_level_attentions = self.attention(code_embeddings.transpose(0, 1), encodings.transpose(0, 1), encodings.transpose(0, 1), key_padding_mask=key_padding_mask)
        nq, _, emb_dim = contextvec.shape
        word_level_attentions = word_level_attentions\
            .view(b, 1, ns, nt)\
            .expand(b, nq, ns, nt)
        traceback_word_level_attentions = traceback_word_level_attentions\
            .view(b, 1, ns, nt)\
            .expand(b, nq, ns, nt)
        attention = word_level_attentions*sentence_level_attentions.unsqueeze(3)
        traceback_attention = traceback_word_level_attentions*sentence_level_attentions.unsqueeze(3)
        if self.concatenate_code_embedding:
            encoding = torch.cat([contextvec, code_embeddings.transpose(0, 1)], 2)
            encoding = torch.relu(self.linear3(encoding))
        else:
            encoding = contextvec
        scores = self.linear(encoding)
        return scores.transpose(0, 1).squeeze(2), attention, traceback_attention, contextvec.transpose(0, 1)

def abstract_loss_func(total_num_codes, scores, codes, num_codes, attention, traceback_attention, context_vec, article_sentences_lengths, clustering, labels, attention_sparsity=False, traceback_attention_sparsity=False, gamma=1):
    b, nq, ns, nt = attention.shape
    positive_labels = labels.sum()
    negative_labels = num_codes.sum() - positive_labels
    pos_weight = negative_labels/positive_labels
    losses = F.binary_cross_entropy_with_logits(scores, labels.float(), pos_weight=pos_weight, reduction='none')
    code_mask = (torch.arange(nq, device=labels.device) < num_codes.unsqueeze(1))
    loss = losses[code_mask].mean()*b
    if attention_sparsity:
        loss += gamma*entropy(attention.view(b, nq, ns*nt))[code_mask].mean()*b
    if traceback_attention_sparsity:
        loss += gamma*entropy(traceback_attention.view(b, nq, ns*nt))[code_mask].mean()*b
    return loss

def loss_func_creator(attention_sparsity=False, traceback_attention_sparsity=False, gamma=1):
    def loss_func_wrapper(total_num_codes, scores, codes, num_codes, attention, traceback_attention, context_vec, article_sentences_lengths, clustering, labels):
        return abstract_loss_func(total_num_codes, scores, codes, num_codes, attention, traceback_attention, context_vec, article_sentences_lengths, clustering, labels,
                                  attention_sparsity=attention_sparsity, traceback_attention_sparsity=traceback_attention_sparsity, gamma=gamma)
    return loss_func_wrapper

def statistics_func(total_num_codes, scores, codes, num_codes, attention, traceback_attention, context_vec, article_sentences_lengths, clustering, labels):
    b, nq, ns, nt = attention.shape
    code_mask = (torch.arange(labels.size(1), device=labels.device) < num_codes.unsqueeze(1))
    positives = get_code_counts(total_num_codes, codes, code_mask, (scores > 0))
    true_positives = get_code_counts(total_num_codes, codes, code_mask, ((scores > 0) & (labels == 1)))
    relevants = get_code_counts(total_num_codes, codes, code_mask, labels)
    return {'positives':positives,
            'true_positives':true_positives,
            'relevants':relevants,
            'total_predicted':code_mask.sum(),
            'accuracy_sum':((scores[code_mask] > 0).long() == labels[code_mask]).sum().float()*b/code_mask.sum(),
            'attention_entropy':entropy(attention.view(b, nq, ns*nt))[code_mask].mean()*b,
            'traceback_attention_entropy':entropy(traceback_attention.view(b, nq, ns*nt))[code_mask].mean()*b}

class LinearizedCodesTransformer(nn.Module):
    def __init__(self, num_embeddings, d_model=100, num_layers=6, nhead=4):
        super(LinearizedCodesTransformer, self).__init__()
        self.embeddings = nn.Embedding(num_embeddings, d_model)
        self.positional_encodings = PositionalEncoding(d_model=d_model)
        encoder_layer = nn.TransformerEncoderLayer(d_model=d_model, nhead=nhead, dim_feedforward=4*d_model)
        self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers)
        self.hidden_size = d_model
        self.num_layers = num_layers
        self.num_heads = nhead

    def forward(self, token_ids, attention_mask):
        b, nl, nh, nt = token_ids.size(0), self.num_layers, self.num_heads, token_ids.size(1)
        outputs = self.transformer_encoder(self.positional_encodings(self.embeddings(token_ids).transpose(0, 1)), src_key_padding_mask=~attention_mask).transpose(0, 1)
        return outputs, outputs[0,0,0]*torch.eye(nt).expand(b, nl, nh, nt, nt)

# Taken from https://github.com/pytorch/examples/tree/master/word_language_model
# Temporarily leave PositionalEncoding module here. Will be moved somewhere else.
class PositionalEncoding(nn.Module):
    r"""Inject some information about the relative or absolute position of the tokens
        in the sequence. The positional encodings have the same dimension as
        the embeddings, so that the two can be summed. Here, we use sine and cosine
        functions of different frequencies.
    .. math::
        \text{PosEncoder}(pos, 2i) = sin(pos/10000^(2i/d_model))
        \text{PosEncoder}(pos, 2i+1) = cos(pos/10000^(2i/d_model))
        \text{where pos is the word position and i is the embed idx)
    Args:
        d_model: the embed dim (required).
        dropout: the dropout value (default=0.1).
        max_len: the max. length of the incoming sequence (default=5000).
    Examples:
        >>> pos_encoder = PositionalEncoding(d_model)
    """

    def __init__(self, d_model, dropout=0.1, max_len=5000):
        super(PositionalEncoding, self).__init__()
        self.dropout = nn.Dropout(p=dropout)

        pe = torch.zeros(max_len, d_model)
        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)
        pe = pe.unsqueeze(0).transpose(0, 1)
        self.register_buffer('pe', pe)

    def forward(self, x):
        r"""Inputs of forward function
        Args:
            x: the sequence fed to the positional encoder model (required).
        Shape:
            x: [sequence length, batch size, embed dim]
            output: [sequence length, batch size, embed dim]
        Examples:
            >>> output = pos_encoder(x)
        """

        x = x + self.pe[:x.size(0), :]
        return self.dropout(x)


"""
This module extracts features from tokenized input sequences.
"""

import torch
from transformers import AutoModel, AutoTokenizer, pipeline
import re
import string
import numpy as np
import pandas as pd
from collections import Counter
from datetime import datetime, timedelta
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import LatentDirichletAllocation
from preprocess import Preprocessor
import pickle
import csv
import argparse
import os
from nltk import pos_tag

emojiPattern = re.compile("["
	u"\U0001F600-\U0001F64F" # facial emojis
	u"\U0001F300-\U0001F5FF" # symbol emojis
	u"\U0001F680-\U0001F6FF" # transport and map emojis
	u"\U0001F1E0-\U0001F1FF" # flags (possibly iOS-only)
"]+", flags=re.UNICODE)

emoticonPattern = re.compile(
	r"(\:\w+\:|\<[\/\\]?3|[\(\)\\\D|\*\$][\-\^]?[\:\;\=]|[\:\;\=B8][\-\^]?[3DOPp\@\$\*\\\)\(\/\|])(?=\s|[\!\.\?]|$)"
)

linkPattern = re.compile(
	r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+"
)

codeSnippetPattern = re.compile(r"```[\s\S]*?```")

hashtagPattern = re.compile(r"#[^\s]+")

abbreviationsPattern = re.compile(
	r"\b(?:[a-z]*[A-Z][a-z]*(?:[A-Z][a-z]*)?|[A-Z]+(?![a-z]))\b"
)

customDscEmojiPattern = re.compile(r"<(a)?:\w+:\d+>")

class FeatExtractor:
	def __init__(self, mdl, nTopics = 10):
		self.sentPipeline = pipeline("sentiment-analysis", model=mdl)
		self.vectorizer = TfidfVectorizer(stop_words="english")
		self.lda = LatentDirichletAllocation(n_components=nTopics, random_state=69)

	def extract_feats(self, msgs, userID=None):
		if not msgs:
			return []
		if not msgs:
			return []

		msgs = self.removeDscEmoji(msgs)

		feats = {
			#0 current userEmb dimensionality: 12
			"meanWordcount": self.mean_wordcount(msgs),
			"vocabRichness": self.vocab_richness(msgs),
			"meanEmojiCount": self.mean_emoji_count(msgs),
			"meanEmoticonCount": self.mean_emoticon_count(msgs),
			"meanPunctuationCount": self.mean_punctuation_count(msgs),
			"meanSentimentScore": self.mean_sentiment_score(msgs),
			#"dominantTopics": self.dominant_topics(msgs),
			#"meanResponseTime": self.mean_response_time(msgs),
			#"meanMsgCountPerDay": self.mean_msg_count_per_day(msgs),
			"meanLinksPerMsg": self.mean_links_per_msg(msgs),
			#"meanCodeSnippetsPerMsg": self.mean_code_snippets_per_msg(msgs),
			#"dominantWritingStyles": self.dominant_writing_styles(msgs),
			"meanAbbreviationsPerMsg": self.mean_abbreviations_per_msg(msgs),
			#"meanHashtagCountPerMsg": self.mean_hashtag_count_per_msg(msgs),
			"meanSentenceLen": self.mean_sentence_len(msgs),
			"meanNounFreq": self.mean_noun_freq(msgs),
			"meanVerbFreq": self.mean_verb_freq(msgs),
			"meanAdjFreq": self.mean_adjective_freq(msgs)
			#"meanAttachmentCountPerMsg": self.mean_attachment_count_per_msg(msgs),
			#"meanQuoteCountPerMsg": self.mean_quote_count_per_msg(msgs),
			#"dominantActiveHours": self.dominant_active_hours(msgs)
		}

		for k, v in feats.items():
			if v is None:
				print(f"Feature {k} is None for user {userID}")
				raise ValueError
		return feats if all(v is not None for v in feats.values()) else []

	def save_feats_for_later(self, feats, userID, path):
		with open(path, "wb") as f:
			pickle.dump(feats, f)

		with open(path + ".csv", "a") as f:
			writer = csv.writer(f)
			writer.writerow([userID] + feats)

	def load_feats(self, path):
		with open(path, "rb") as f:
			feats = pickle.load(f)
		return feats

	def read_msgs_from_file(self, path):
		with open(path, "r", encoding="utf8") as f:
			#fieldNames = ["uid", "timestamp", "content", "sentiment"]
			fieldNames = ["uid", "content", "sentiment"]
			reader = csv.DictReader(f, fieldnames=fieldNames, delimiter="|")
			next(reader)
			header = reader.fieldnames
			print(header)
			if header != fieldNames:
				raise ValueError("Invalid header")
			msgs = [row for row in reader if any(row.values())]
			for msg in msgs:
				msg.pop(None, None)
		# return format: [{"uid": "user#1234", "content": "hello world", "sentiment": "pos"}, ...]
		return msgs

	def group_msgs_by_user(self, msgs):
		groups = {}
		for msg in msgs:
			uid = msg["uid"]
			content = msg["content"]
			if uid not in groups:
				groups[uid] = []
			groups[uid].append(content)
		return groups

	def txt_to_csv(self, txtPath, csvPath, soughtUser):
		pattern = r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6}):: .+\/(.+?): (.+)"
		alsoPattern = r"^(\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2})\s::\s([\w#]+):\s(.+)$"
		with open(txtPath, "r") as infile, open(csvPath, "w") as outfile:
			writer = csv.DictWriter(outfile, fieldnames=["uid", "timestamp", "content"], delimiter="|")
			writer.writeheader()

			for line in infile:
				match = re.match(pattern, line)
				alsoMatch = re.match(alsoPattern, line)
				if match:
					timestamp, discordTag, content = match.groups()
					if "The Butler#5636" not in discordTag and "MarkTheBot#5636" not in discordTag:
						writer.writerow({"uid": discordTag, "timestamp": timestamp, "content": content})
				elif alsoMatch:
					timestamp, discordTag, content = alsoMatch.groups()
					if "The Butler#5636" not in discordTag and "MarkTheBot#5636" not in discordTag:
						writer.writerow({"uid": discordTag, "timestamp": timestamp, "content": content})
		print(f"Saved {csvPath}")

	def cleanup_csv(self, csvPath):
		raise NotImplementedError

	def get_required_fields(self):
		featMethods = [
			self.mean_wordcount,
			self.vocab_richness,
			self.mean_emoji_count,
			self.mean_emoticon_count,
			self.mean_punctuation_count,
			self.mean_sentiment_score,
			self.dominant_topics,
			self.mean_response_time,
			self.mean_msg_count_per_day,
			self.mean_links_per_msg,
			self.mean_code_snippets_per_msg,
			self.dominant_writing_styles,
			self.mean_abbreviations_per_msg,
			self.mean_hashtag_count_per_msg,
			self.mean_attachment_count_per_msg,
			self.mean_quote_count_per_msg,
			self.dominant_active_hours
		]

		requiredFields = set()
		for method in featMethods:
			fields = getattr(method, "requiredFields", [])
			requiredFields.update(fields)

		return list(requiredFields)

	def check_if_fields_exist(self, msgs, requiredFields):
		#if not "uid" in msgs[0] or not "content" in msgs[0] or not "timestamp" in msgs[0]:
		print(f"msgs[0]: {msgs[0]}")
		print(f"msgs: {msgs}")
		if not "uid" in msgs[0] or not "content" in msgs[0]:
			#raise ValueError("Fields 'uid', 'content', and 'timestamp' must be present in the input data!")
			raise ValueError("Fields 'uid' and 'content' must be present in the input data!")

	def remove_none(self, msgs):
		cleaned = []
		for row in msgs:
			if None not in row.values():
				cleaned.append(row)
		msgs = cleaned
		return msgs

	def removeDscEmoji(self, msgs):
		clnMsgs = []
		for msg in msgs:
			cln = customDscEmojiPattern.sub("", msg["content"])
			clnMsg = msg.copy()
			clnMsg["content"] = cln
			clnMsgs.append(clnMsg)
			msgs = clnMsgs
		return msgs

	def extract_and_store_feats(self, msgs, userID=None, path="./pickles/"):
		requiredFields = self.get_required_fields()
		#self.check_if_fields_exist(msgs, requiredFields) # disabled due to restructuring of msgs
		groupedMsgs = self.group_msgs_by_user(msgs)

		if userID is not None:
			#if userID not in groupedMsgs or len(groupedMsgs[userID]) < 25:
			if len(msgs) < 25:
				return None
			#msgs = groupedMsgs[userID] # disabled for now
			assert all(msg["uid"] == userID for msg in msgs), f"msgs contains messages from multiple users: {[msg['uid'] for msg in msgs]}" #sanity check to ensure that all dicts in msgs belong to the same user
			feats = self.extract_feats(msgs=msgs, userID=userID)
			if not feats:
				print(f"{userID} has 0 feats!")
				return None
			else:
				print(f"Extracted {len(feats)} features from {userID}")
				return feats

		else:
			feats = {}
			for uid, userMsgs in groupedMsgs.items():
				if len(userMsgs) < 25:
					print(f"Skipping {uid} because they have less than 25 messages")
					continue
				feats[uid] = self.extract_feats(msgs=userMsgs, userID=uid)
				print(f"Extracted {len(feats)} features from {uid}")

				picklePath = os.path.join(path, f"{uid.split('#')[0]}_feats.pkl") # using only the first part of the uid as the filename
				self.save_feats_for_later(feats[uid], userID, picklePath)
			return feats

	### Feature extraction methods ###

	def mean_wordcount(self, msgs):
		print("Extracting mean wordcount...")
		return round(np.mean([len(msg["content"].split()) for msg in msgs]), 2)
	mean_wordcount.requiredFields = ["content"]

	def vocab_richness(self, msgs):
		print("Extracting vocabulary richness...")
		words = [word for msg in msgs for word in msg["content"].split()]
		uniqueWords = len(set(words))
		if len(words) == 0:
			return 0
		else:
			return round(uniqueWords / len(words), 2)
	vocab_richness.requiredFields = ["content"]

	def mean_emoji_count(self, msgs):
		print("Extracting mean emoji count...")
		emojiCount = [len(emojiPattern.findall(msg["content"])) for msg in msgs]
		return round(np.mean(emojiCount), 2)
	mean_emoji_count.requiredFields = ["content"]

	def mean_emoticon_count(self, msgs):
		print("Extracting mean emoticon count...")
		emoticonCount = [len(emoticonPattern.findall(msg["content"])) for msg in msgs]
		return round(np.mean(emoticonCount), 2)
	mean_emoticon_count.requiredFields = ["content"]

	def mean_punctuation_count(self, msgs):
		print("Extracting mean punctuation count...")
		punctCount = [sum(char in string.punctuation for char in msg["content"]) for msg in msgs]
		return round(np.mean(punctCount), 2)
	mean_punctuation_count.requiredFields = ["content"]

	def mean_sentiment_score(self, msgs):
		print("Extracting mean sentiment score...")
		sentMapping = {"pos": 2, "neg": 0, "neu": 1} # adjusted due to CrossEntropyLoss
		msgs = self.remove_none(msgs)
		if not msgs or len(msgs[0]["content"].split()) < 1:
			print(f"Empty msgs!")
			return None
		sentimentScores = []
		for row in msgs:
			try:
				sentimentScore = sentMapping.get(row.get("sentiment"), 0) if len(row["content"].split()) >= 1 else 0
				sentimentScores.append(sentimentScore)
			except KeyError as e:
				print(f"KeyError: at {row}")
				raise e
		return round(np.mean(sentimentScores), 2)
	mean_sentiment_score.requiredFields = ["sentiment"]

	def dominant_topics(self, msgs):
		print("Extracting dominant topics...")
		msgTexts = [msg["content"] for msg in msgs]
		docTermMatrix = self.vectorizer.fit_transform(msgTexts)
		self.lda.fit(docTermMatrix)
		topicDistr = np.mean(self.lda.transform(docTermMatrix), axis=0)
		return int(topicDistr.argmax())
	dominant_topics.requiredFields = ["content"]

	def mean_response_time(self, msgs):
		print("Extracting mean response time...")
		timestamps = [datetime.fromisoformat(msg["timestamp"]) for msg in msgs]
		responseTimes = [timestamps[i + 1] - timestamps[i] for i in range(len(timestamps) - 1)]
		responseTimes = [rt for rt in responseTimes if rt < timedelta(hours=1)]
		if not responseTimes:
			return 0.0
		return round(np.mean(responseTimes).total_seconds(), 2)
	mean_response_time.requiredFields = ["timestamp"]

	def mean_msg_count_per_day(self, msgs):
		print("Extracting mean message count per day...")
		timestamps = [datetime.fromisoformat(msg["timestamp"]).date() for msg in msgs]
		if len(timestamps) < 2 or timestamps[-1] == timestamps[0] or not timestamps:
			return 0.0
		days = (timestamps[-1] - timestamps[0]).days
		return round(len(timestamps) / days, 2)
	mean_msg_count_per_day.requiredFields = ["timestamp"]

	def mean_links_per_msg(self, msgs):
		print("Extracting mean links per message...")
		linkCount = [len(linkPattern.findall(msg["content"])) for msg in msgs]
		return round(np.mean(linkCount), 2)
	mean_links_per_msg.requiredFields = ["content"]

	def mean_code_snippets_per_msg(self, msgs):
		print("Extracting mean code snippets per message...")
		codeSnippetCount = [len(codeSnippetPattern.findall(msg["content"])) for msg in msgs]
		return round(np.mean(codeSnippetCount), 2)
	mean_code_snippets_per_msg.requiredFields = ["content"]

	def dominant_writing_styles(self, msgs):
		print("Extracting dominant writing styles...")
		raise NotImplementedError
	dominant_writing_styles.requiredFields = ["content"]

	def mean_abbreviations_per_msg(self, msgs):
		print("Extracting mean abbreviations per message...")
		abbreviationsCount = [len(abbreviationsPattern.findall(msg["content"])) for msg in msgs]
		return round(np.mean(abbreviationsCount), 2)
	mean_abbreviations_per_msg.requiredFields = ["content"]

	def mean_hashtag_count_per_msg(self, msgs):
		print("Extracting mean hashtag count per message...")
		hashtagCount = [len(hashtagPattern.findall(msg["content"])) for msg in msgs]
		return round(np.mean(hashtagCount), 2)
	mean_hashtag_count_per_msg.requiredFields = ["content"]

	def mean_sentence_len(self, msgs):
		print("Extracting mean sentence length...")
		content = " ".join([msg["content"] for msg in msgs])
		sentences = re.findall(r'\w[^.!?]*[.!?]', content)
		nSentences = len(sentences)
		nWords = len(re.findall(r'\w+', content))
		try:
			return nWords / nSentences
		except ZeroDivisionError:
			return 0
	mean_sentence_len.requiredFields = ["content"]

	def mean_noun_freq(self, msgs):
		print("Extracting mean noun frequency...")
		words = [word for msg in msgs for word in msg["content"].split()]
		tagged = pos_tag(words)
		pos = Counter(tag for _, tag in tagged)
		try:
			return pos["NN"] / sum(pos.values())
		except ZeroDivisionError:
			return 0
	mean_noun_freq.requiredFields = ["content"]

	def mean_verb_freq(self, msgs):
		print("Extracting mean verb frequency...")
		words = [word for msg in msgs for word in msg["content"].split()]
		tagged = pos_tag(words)
		pos = Counter(tag for _, tag in tagged)
		try:
			return pos["VB"] / sum(pos.values())
		except ZeroDivisionError:
			return 0
	mean_verb_freq.requiredFields = ["content"]

	def mean_adjective_freq(self, msgs):
		print("Extracting mean adjective frequency...")
		words = [word for msg in msgs for word in msg["content"].split()]
		tagged = pos_tag(words)
		pos = Counter(tag for _, tag in tagged)
		try:
			return pos["JJ"] / sum(pos.values())
		except ZeroDivisionError:
			return 0
	mean_adjective_freq.requiredFields = ["content"]

	def mean_attachment_count_per_msg(self, msgs):
		print("Extracting mean attachment count per message...")
		attachmentCount = [len(msg["attachments"]) for msg in msgs]
		return np.mean(attachmentCount)
	mean_attachment_count_per_msg.requiredFields = ["attachments"]

	def mean_quote_count_per_msg(self, msgs):
		print("Extracting mean quote count per message...")
		quoteCount = [len(msg["attachments"].split(",")) if msg["attachments"] else 0 for msg in msgs]
		return np.mean(quoteCount)
	mean_quote_count_per_msg.requiredFields = ["attachments"]

	def dominant_active_hours(self, msgs):
		print("Extracting dominant active hours...")
		raise NotImplementedError
	dominant_active_hours.requiredFields = ["timestamp"]

if __name__ == "__main__":
	parser = argparse.ArgumentParser()
	parser.add_argument("--uid", type=str, required=False, help="UID of the user for all messages")
	args = parser.parse_args()

	mdl = "distilbert-base-uncased"
	extractor = FeatExtractor(mdl)

	if os.path.splitext("./datasets/messagesNew.txt")[1] == ".txt":
		extractor.txt_to_csv("./datasets/messagesNew.txt", "./datasets/msgsNew.csv", args.uid)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/10/15 12:34
# @Author  : Zhe Jun
# @File    : US08.py
# @Software: PyCharm
from datetime import datetime
import re

def birth_before_parents_marriage(indi_list, fam_list):

    for fam in fam_list:

        if fam['CHIL'] != 'NONE' and fam['MARR'] != 'NONE':
            marr = datetime.strptime(fam['MARR'], '%Y-%m-%d')

            for child in fam['CHIL']:
                find_child_index = int(re.sub('\D', '', child)) - 1
                birt = datetime.strptime(indi_list[find_child_index]['BIRT'], '%Y-%m-%d')

                if marr > birt:
                    print("ERROR: INDIVIDUAL: US08: {}: {}: Birth {} before parents marriage {}".format( indi_list[find_child_index]['num']+4, indi_list[find_child_index]['INDI'], birt, marr))
                    return False

    return True


# 서로소 집합(disjoint set)
parent = {}
rank = {}

def make_set(v):
    parent[v] = v
    rank[v] = 0

def find(v):
    if parent[v] != v:
        parent[v] = find(parent[v])

    return parent[v]

def union(v, u):
    r1 = find(v)
    r2 = find(u)

    if r1 != r2:
        if rank[r1] > rank[r2]:
            parent[r2] = r1
        else:
            parent[r1] = r2

            if rank[r1] == rank[r2]:
                rank[r2] += 1

# 크러스컬 알고리듬(Kruskal’s algorithm)
def solution(n, costs):
    # 간선 가중치 오름차순으로 정렬
    costs.sort(key=lambda x: x[2])

    # 정점 트리
    for node in range(n):
        make_set(node)

    # 서로소 집합으로 MST에 포함되는 간선인지 판단
    result = 0

    for a, b, c in costs:
        if find(a) != find(b):
            union(a, b)
            result += c

    return result

# 정확성  테스트
# 테스트 1 〉	통과 (0.01ms, 10.2MB)
# 테스트 2 〉	통과 (0.01ms, 10.2MB)
# 테스트 3 〉	통과 (0.03ms, 10.3MB)
# 테스트 4 〉	통과 (0.04ms, 10.3MB)
# 테스트 5 〉	통과 (0.02ms, 10.3MB)
# 테스트 6 〉	통과 (0.06ms, 10.2MB)
# 테스트 7 〉	통과 (0.06ms, 10.2MB)
# 테스트 8 〉	통과 (0.02ms, 10.3MB)
# 채점 결과
# 정확성: 100.0
# 합계: 100.0 / 100.0

assert (solution(4, [[0, 1, 1], [0, 2, 2], [1, 2, 5], [1, 3, 1], [2, 3, 8]]) == 4)


from AlbotOnline.Snake.SnakeGame import SnakeGame
import random
import copy

FREE = 0
BLOCKED = 1
PLAYER = 2
ENEMY = 3
DIM = 10
matrix = [[0 for y in range(DIM)] for x in range(DIM)]

MAX_DEPTH = 4
def printBoard(matrix):
    for y in range(DIM):
        for x in range(DIM):
            print(matrix[x][y], end=" ")
        print()
    print()

def isValidMove(matrix, simulatedX, simulatedY):
    isValid = simulatedX < 10 and simulatedX >= 0 and simulatedY < 10 and simulatedY >= 0 and matrix[simulatedX][simulatedY] == FREE
   #print('isValid', isValid, simulatedX, simulatedY)
    #printBoard(matrix)
    return isValid

def evaluteAgent(board, depth, simulatedX, simulatedY):
    sum = 0
    #right
    for x in range(simulatedX, DIM):
        cell = board[x][simulatedY]
        if cell == FREE:
            sum += 0.5
        else:
            break
    #left
    for x in range(simulatedX, 0, -1):
        cell = board[x][simulatedY]
        if cell == FREE:
            sum += 0.5
        else:
            break
    #down
    for y in range(simulatedY, DIM):
        cell = board[simulatedX][y]
        if cell == FREE:
            sum += 0.5
        else:
            break
    #up
    for y in range(simulatedY, 0, -1):
        cell = board[simulatedX][y]
        if cell == FREE:
            sum += 0.5
        else:
            break
    if matrix[simulatedX][simulatedY] != FREE:
       #print('occupied', simulatedX, simulatedY, depth)
        sum = -10
    depthFactor = MAX_DEPTH - depth + 1
    return sum * (depthFactor/MAX_DEPTH+1)

def evaluateBoard(board, depth, simulatedPlayerX, simulatedPlayerY, simulatedEnemyX, simulatedEnemyY):
    return evaluteAgent(board, depth, simulatedPlayerX, simulatedPlayerY)

def simulateMove(matrix, currentX, currentY, move, type):
    if move == "left":
        return (currentX-1, currentY)
    elif move == "right":
        return (currentX+1, currentY)
    elif move == "up":
        return (currentX, currentY-1)
    elif move == "down":
        return (currentX, currentY+1)

def getDirection(possibleMove):
    allMoves = ["left", "right", "down", "up"]
    for possibleMove in getPossibleMoves:
        allMoves.remove(possibleMove)
    return allMoves[0]

def nextState(depth, matrix, currentSum, moves, currentPlayerX, currentPlayerY, currentEnemyX, currentEnemyY):
   #print("moves", moves)
    if depth == 0:
        return (moves, currentSum)
    nextStates = []
    allMoves = ["left", "right", "down", "up"]
    playerMoves = allMoves
    enemyMoves = allMoves
    for playerMove in playerMoves:
        nextMatrix = copy.deepcopy(matrix)
        nextPlayerX, nextPlayerY = simulateMove(nextMatrix, currentPlayerX, currentPlayerY, playerMove, PLAYER)
        nextMatrix[currentPlayerX][currentPlayerY] = BLOCKED

       #print('withOUT nextMove', moves)
        withNextMove = copy.deepcopy(moves) + [playerMove]
       #print('with nextMove', withNextMove)
        if not isValidMove(nextMatrix, nextPlayerX, nextPlayerY):
            continue
        for enemyMove in enemyMoves:
            nextNextMatrix = copy.deepcopy(nextMatrix)
            nextEnemyX, nextEnemyY = simulateMove(nextNextMatrix, currentEnemyX, currentEnemyY, enemyMove, ENEMY)
            nextNextMatrix[currentEnemyX][currentEnemyY] = BLOCKED
            if isValidMove(matrix, nextEnemyX, nextEnemyY):
                nextStates.append(nextState(depth-1, nextNextMatrix, currentSum, withNextMove, nextPlayerX, nextPlayerY, nextEnemyX, nextEnemyY))
   #print("Nextstates")
   #print(nextStates)
   #print("depth", MAX_DEPTH-depth)
    #printBoard(matrix)

    if len(nextStates) > 0:
        maxTuple = max(nextStates, key=lambda pair: pair[1])
       #print("maxTuple", maxTuple)
        evaluation = evaluateBoard(copy.deepcopy(matrix), depth, currentPlayerX, currentPlayerY, currentEnemyX, currentEnemyY)
        return (maxTuple[0], maxTuple[1] + evaluation)
    else:
       #print("no next state")
       #print(moves, currentSum, depth)
        return (moves, currentSum)

game = SnakeGame()

global previousPlayerX
previousPlayerX = None

global previousPlayerY
previousPlayerY = None

global previousEnemyX
previousEnemyX = None

global previousEnemyY
previousEnemyY = None

def addGlobalMove(currentX, currentY, type):
    global previousPlayerX
    global previousPlayerY
    global previousEnemyX
    global previousEnemyY
    matrix[currentX][currentY] = type
    if type == PLAYER:
        if previousPlayerX and previousPlayerY:
            matrix[previousPlayerX][previousPlayerY] = BLOCKED
        previousPlayerX = currentX
        previousPlayerY = currentY

    elif type == ENEMY:
        if previousEnemyX and previousEnemyY:
            matrix[previousEnemyX][previousEnemyY] = BLOCKED
        previousEnemyX = currentX
        previousEnemyY = currentY

while(game.awaitNextGameState() == "ongoing"):
    board = game.currentBoard
    # board.printBoard("Current Board")
    #addMoves()
    currentPlayerX = board.player.x
    currentPlayerY = board.player.y
    addGlobalMove(currentPlayerX, currentPlayerY, PLAYER)

    currentEnemyX = board.enemy.x
    currentEnemyY = board.enemy.y
    addGlobalMove(currentEnemyX, currentEnemyY, ENEMY)
   #print("new state")
    #printBoard(matrix)
   #print(currentPlayerX, currentPlayerY)
   #print(currentEnemyX, currentEnemyX)

    moves, value = nextState(MAX_DEPTH, copy.deepcopy(matrix), 0, [], currentPlayerX, currentPlayerY, currentEnemyX, currentEnemyY)
   #print("while loop moves", moves)
   #print()

    #print(game.evaluateBoard(board))
    #print(board.blocked)
    #print(board.player.x)
    #print(board.enemy.x)
    if (len(moves) == 0):
        moves, value = nextState(1, copy.deepcopy(matrix), 0, [], currentPlayerX, currentPlayerY, currentEnemyX, currentEnemyY)

    playerMoves, enemyMoves = game.getPossibleMoves(board)

    allMoves = ["left", "right", "down", "up"]
    nextMatrix = copy.deepcopy(matrix)
    if len(moves) == 0:
        for move in allMoves:
            nextX, nextY = simulateMove(nextMatrix, currentPlayerX, currentPlayerY, move, PLAYER)
            print(nextX, nextY)
            if isValidMove(nextMatrix, nextX, nextY):
                game.makeMove(move)
                break
    else:
        game.makeMove(moves[0])


import sys
input = sys.stdin.readline

n, m = map(int, input().split())
paper = []
ans = 0
for _ in range(n):
    paper.append(list(map(int, input().rstrip())))

for i in range(2 ** (n*m)):
    score = 0
    for r in range(n):
        rowsum = 0
        for c in range(m):
            idx = r * m + c
            if i & (1 << idx):
                rowsum = rowsum * 10 + paper[r][c]
            else:
                score += rowsum
                rowsum = 0
        score += rowsum

    for c in range(m):
        colsum = 0
        for r in range(n):
            idx = r * m + c
            if i & (1 << idx) == 0:
                colsum = colsum * 10 + paper[r][c]
            else:
                score += colsum
                colsum = 0
        score += colsum

    ans = max(ans, score)

print(ans)


import sys

import os
from mqtt.client.factory import MQTTFactory
from twisted.application.internet import ClientService
from twisted.internet.defer import inlineCallbacks
from twisted.internet.endpoints import clientFromString
from twisted.internet import reactor

from twisted.logger import Logger, LogLevel, globalLogBeginner, textFileLogObserver, \
    FilteringLogObserver, LogLevelFilterPredicate

# ----------------
# Global variables
# ----------------
from ledslie.config import Config
from ledslie.definitions import LEDSLIE_TOPIC_STATS_BASE, LEDSLIE_TOPIC_SEQUENCES_PROGRAMS

logLevelFilterPredicate = LogLevelFilterPredicate(defaultLogLevel=LogLevel.info)

log = Logger(__file__.split(os.sep)[-1])

# -----------------
# Utility Functions
# -----------------

def startLogging(console=True, filepath=None):
    '''
    Starts the global Twisted logger subsystem with maybe
    stdout and/or a file specified in the config file
    '''
    global logLevelFilterPredicate

    observers = []
    if console:
        observers.append(FilteringLogObserver(observer=textFileLogObserver(sys.stdout),
                                              predicates=[logLevelFilterPredicate]))

    if filepath is not None and filepath != "":
        observers.append(FilteringLogObserver(observer=textFileLogObserver(open(filepath, 'a')),
                                              predicates=[logLevelFilterPredicate]))
    globalLogBeginner.beginLoggingTo(observers)

def setLogLevel(namespace=None, levelStr='info'):
    '''
    Set a new log level for a given namespace
    LevelStr is: 'critical', 'error', 'warn', 'info', 'debug'
    '''
    level = LogLevel.levelWithName(levelStr)
    logLevelFilterPredicate.setLogLevelForNamespace(namespace=namespace, level=level)

def CreateContent(contentCls):
    config = Config()
    startLogging()
    log_level = 'info'
    if config['DEBUG']:
        log_level = 'debug'
    setLogLevel(namespace='mqtt', levelStr=log_level)
    setLogLevel(namespace=contentCls.__name__, levelStr=log_level)

    factory = MQTTFactory(profile=MQTTFactory.PUBLISHER)
    myEndpoint = clientFromString(reactor, Config().get('MQTT_BROKER_CONN_STRING'))
    serv = contentCls(myEndpoint, factory)
    serv.startService(contentCls.__name__)
    return serv

class GenericContent(ClientService):
    def __init__(self, endpoint, factory, reactor=None):
        super().__init__(endpoint, factory)
        if reactor is None:
            from twisted.internet import reactor
        self.reactor = reactor
        self.config = Config()
        self.protocol = None
        self._system_name = None

    def startService(self, name):
        log.info("starting MQTT Content Publisher Service")
        # invoke whenConnected() inherited method
        self._system_name = name
        self.whenConnected().addCallback(self.connectToBroker)
        ClientService.startService(self)

    @inlineCallbacks
    def connectToBroker(self, protocol):
        '''
        Connect to MQTT broker
        '''
        self.protocol = protocol
        self.protocol.onDisconnection = self.onDisconnection
        self.protocol.setWindowSize(3)
        try:
            yield self.protocol.connect(self._system_name, keepalive=60)
        except Exception as e:
            self.log.error("Connecting to {broker} raised {excp!s}",
                      broker=self.config.get('MQTT_BROKER_CONN_STRING'), excp=e)
        else:
            self.log.info("Connected to {broker}", broker=self.config.get('MQTT_BROKER_CONN_STRING'))
            self.reactor.callLater(0, self.onBrokerConnected)
        self_name = self.__class__.__name__
        self.publish(topic=LEDSLIE_TOPIC_STATS_BASE+self_name, message="%s now (re-)connected" % self_name)

    def onBrokerConnected(self):
        log.info("onBrokerConnected called")

    def onDisconnection(self, reason):
        '''
        get notfied of disconnections
        and get a deferred for a new protocol object (next retry)
        '''
        log.debug("<Connection was lost !> <reason={r}>", r=reason)
        self.whenConnected().addCallback(self.connectToBroker)

    def publish(self, topic, message, qos=0, retain=False):
        if hasattr(message, 'serialize'):
            message = message.serialize()
        self.log.debug("To '{topic}', Published: '{data}'", topic=topic, data=message)
        return self.protocol.publish(topic, message, qos, retain)

    def remove_display(self, program_name):
        """
        Remove the program from the display.
        :param program_name: The name of the program to remove
        :type program_name: str
        :return: The deferred that's called when the command has been send.
        :rtype: deferred
        """
        return self.publish(LEDSLIE_TOPIC_SEQUENCES_PROGRAMS[:-1] + program_name, "")


"""
A Python implementation of the FIFO, LRU and OPT page replacement algorithms.

Author: Jaron Cohen
Student Number: CHNJAR003

Sources:
[CSC3002F, 2020] CSC3002F (2020). A Comparison of Three Page Replacement Algorithms:
FIFO, LRU and Optimal. Department of Computer Science, University of Cape Town,
South Africa.
[Silberschatz et al., 2018] Silberschatz, A., Galvin, P., and Gagne, G. (2018). Operating Sys-
tem Concepts - 10th Edition. John Wiley & Sons, New York, USA.

"""

import sys
import random
from collections import deque

"""
Performs the FIFO algorithm on an array of integer page references in the range of 0 to 9 for a given memory size.
parameters:
size - integer number in range 1 to 7 of number of frames in memory
pages - array of integer page references with each element in the range of 0 to 9
"""

def FIFO(size, pages):
    memory = []
    queue = deque([])
    numPageFaults = 0

    for page in pages:
        if len(memory) < size:
            memory.append(page)
            queue.append(page)
            numPageFaults += 1
        else:
            if page not in memory:
                pageToRemove = queue.popleft()
                indexOfPageToRemove = memory.index(pageToRemove)
                memory.remove(pageToRemove)
                memory.insert(indexOfPageToRemove, page)
                queue.append(page)
                numPageFaults += 1
    return numPageFaults

"""
Performs the LRU algorithm on an array of integer page references in the range of 0 to 9 for a given memory size.
parameters:
size - integer number in range 1 to 7 of number of frames in memory
pages - array of integer page references with each element in the range of 0 to 9
"""

def LRU(size, pages):
    memory = []
    stack = deque()
    numPageFaults = 0

    for page in pages:
        if len(memory) < size:
            memory.append(page)
            stack.appendleft(page)
            numPageFaults += 1

        else:
            if page in memory:  # If page is referenced, remove and place on top of stack

                pageStackIndex = stack.index(page)

                del stack[pageStackIndex]
                stack.appendleft(page)

            else:

                minPage = stack.pop()

                indexOfPageToRemove = memory.index(minPage)

                memory.remove(minPage)
                memory.insert(indexOfPageToRemove, page)
                stack.appendleft(page)
                numPageFaults += 1

    return numPageFaults

"""
Performs the OPT algorithm on an array of integer page references in the range of 0 to 9 for a given memory size.
parameters:
size - integer number in range 1 to 7 of number of frames in memory
pages - array of integer page references with each element in the range of 0 to 9
"""

def OPT(size, pages):
    memory = []
    numPageFaults = 0
    for i in range(len(pages)):

        if len(memory) < size:
            memory.append(pages[i])
            numPageFaults += 1

        else:

            if pages[i] not in memory:

                lastToBeUsed = -1
                # Find page in memory that is never referenced in the future
                for frame in memory:
                    # If there is a page in a frame in memory that doesn't appear in the rest of the pages sequence
                    if frame not in pages[i+1:]:
                        lastToBeUsed = frame
                        break

                # If found page not referenced in future, replace it
                if lastToBeUsed != -1:

                    indexOfPageToRemove = memory.index(lastToBeUsed)

                    memory.remove(lastToBeUsed)
                    memory.insert(indexOfPageToRemove, pages[i])
                    numPageFaults += 1
                else:  # If all pages in memory are referenced in future, find farthest in future

                    farthestIndex = pages[i+1:].index(memory[0])
                    pageWithFarthestIndex = memory[0]
                    # Iterate through memory frames and find farthest one in sequence
                    for frameIndex in range(1, len(memory)):
                        newIndex = pages[i+1:].index(memory[frameIndex])
                        if newIndex > farthestIndex:
                            farthestIndex = newIndex
                            pageWithFarthestIndex = memory[frameIndex]
                    indexOfPageToRemove = memory.index(pageWithFarthestIndex)

                    memory.remove(pageWithFarthestIndex)
                    memory.insert(indexOfPageToRemove, pages[i])
                    numPageFaults += 1

    return numPageFaults

def main():

    if len(sys.argv) == 2:
        numFrames = int(sys.argv[1])
        numPages = random.randint(10, 100)
        pages = [random.randint(0, 9) for ranNum in range(numPages)]
        print('Random Page References: ', pages)
        print('Number of Page References: ', len(pages))
        print('Number of Frames: ', numFrames)
        print('FIFO', FIFO(numFrames, pages), 'page faults.')
        print('LRU', LRU(numFrames, pages), 'page faults.')
        print('OPT', OPT(numFrames, pages), 'page faults.')
    else:
        numFrames = int(sys.argv[1])
        try:
            numPages = int(sys.argv[2])
        except:
            print("Invalid second argument provided.")

        pages = [random.randint(0, 9) for ranNum in range(numPages)]
        print('Random Page References: ', pages)
        print('Number of Page References: ', len(pages))
        print('Number of Frames: ', numFrames)
        print('FIFO', FIFO(numFrames, pages), 'page faults.')
        print('LRU', LRU(numFrames, pages), 'page faults.')
        print('OPT', OPT(numFrames, pages), 'page faults.')

if __name__ == "__main__":

    if len(sys.argv) < 2:
        print('Usage: python3 paging.py number_of_frames [number_of_pages]')
        print('Refer to README for additional information.')
    elif len(sys.argv) >= 2 and len(sys.argv) <= 3:

        numFrames = int(sys.argv[1])
        if numFrames in range(1, 8):
            main()
        else:
            print("Invalid number of frames entered. Provide integer from 1 to 7.")

    else:
        print(
            'Too many arguments provided. Refer to README for valid command line arguments.')


#!/usr/bin/env python
from collections import defaultdict
from argparse import ArgumentParser

parser = ArgumentParser()
parser.add_argument("-w", "--week", type=int, required=False)
parser.add_argument("-t", "--todoist", action='store_true', required=False)
args = parser.parse_args()

def todoist_format(week, day, exercise):
    parts = exercise.split(' ')
    duration = 0
    durations = defaultdict(lambda : 0)
    for p in parts:
        if 'x' in p:
            rep, ex = p.split('x')
            rep = int(rep)
            ex = ex.strip('[]')
            for p2 in ex.split(","):
                letter = p2[-1]
                nums = float(p2[:-1])
                duration += rep * nums
                durations[letter] += rep * nums
        else:
            letter = p[-1]
            nums = float(p[:-1])
            duration += nums
            durations[letter] += nums
    ' '.join([f"{mins}{l}" for l, mins in durations.items()])
    print(f"w{week}.{day} -- 5m warm + `{exercise}` + 5m cool -> `{duration + 10}m`") #, -- {durns}")

c25k = {
    1: {
        1: "8x[1j,1.5w]",
        2: "8x[1j,1.5w]",
        3: "8x[1j,1.5w]",
    },
    2: {
        1: "6x[1.5j,2w]",
        2: "6x[1.5j,2w]",
        3: "6x[1.5j,2w]",
    },
    3: {
        1: "2x[1.5j,1.5w,3j,3w]",
        2: "2x[1.5j,1.5w,3j,3w]",
        3: "2x[1.5j,1.5w,3j,3w]",
    },
    4: {
        1: "1x[3j,1.5w,5j,2.5w,3j,1.5w]",
        2: "1x[3j,1.5w,5j,2.5w,3j,1.5w]",
        3: "1x[3j,1.5w,5j,2.5w,3j,1.5w]",
    },
    5: {
        1: "1x[5j,3w,5j,3w,5j]",
        2: "1x[8j,5w,8j]",
        3: "1x[20j]",
    },
    6: {
        1: "1x[5j,3w,8j,3w,5j]",
        2: "1x[10j,3w,10j]",
        3: "1x[22j]",
    },
    7: {
        1: "1x[25j]",
        2: "1x[25j]",
        3: "1x[25j]",
    },
    8: {
        1: "1x[28j]",
        2: "1x[28j]",
        3: "1x[28j]",
    },
    9: {
        1: "1x[30j]",
        2: "1x[30j]",
        3: "1x[30j]",
    },
}

print("Always with 5m warmup and cooldown")
if args.week:
    c25k = {args.week: c25k[args.week]}
for week, days in c25k.items():
    for day, exercise in days.items():
        if args.todoist:
            todoist_format(week, day, exercise)
        else:
            print(f"w{week}.{day} -- {exercise}")


from typing import List
from fastapi.params import Depends
from sqlalchemy.orm.session import Session
from app import models
from fastapi import FastAPI, Response, status, HTTPException, APIRouter
from fastapi.exceptions import RequestValidationError
from .. import schemas
from ..database import SessionLocal, engine, get_db

# create note in database
def create(db: Session, request: schemas.Note):
    new_note = models.Note(title = request.title, note = request.note, owner_id = 1)
    db.add(new_note)
    db.commit()
    db.refresh(new_note)
    return new_note

# get all  notes form database
def read_all(db: Session):
    return db.query(models.Note).all()

# get a single note with id from database
def read(db: Session, id: int):
    return db.query(models.Note).filter(models.Note.id == id).first()

# update note in database
def update(db:Session, id: int, request: schemas.Note):
    updated_note = db.query(models.Note).filter(models.Note.id == id)
    if not updated_note:
        raise HTTPException(status_code=404, detail=f"Cannot find note with id {id}")
    updated_note.update({"title": request.title, "note": request.note})
    db.commit()
    return "updated"

def delete(db:Session, id: int):
    note = db.query(models.Note).filter(models.Note.id == id)
    if not note.first():
        raise HTTPException(status_code=404, detail=f"Cannot find note with id {id}")
    note.delete(synchronize_session=False)
    db.commit()
    return "Deleted"


import os

def FileW(message):

    doc = open("test.txt","w")      # open the file in writing mode
    # "W" is overwrites the file, "A" add the text after the previous one,
    doc.write(message)              # is writing the message contained in the variable
    doc.close()                     # close and save the file
    print('The file has been amended')

def FileR():
    print('almost ready to check the file')
    doc_r = open("test.txt", "r")
    text = doc_r.read()
    position = doc_r.tell()
    doc_r.close()
    print (text)
    print ('there are',position,'charater within the file:')

def Rename():
    confirm = input('Do you want rename the file? Y or N')
    confirm = confirm.lower()
    if confirm == "y":
        newname = input('Add the new filename: ')
        os.rename("test.txt", newname)
    else:
        print("The file name asn't change")

message = input('add a message')
FileW(message)
FileR()
Rename()


import glob
import os
import tarfile

from pygemstones.io import file as f
from pygemstones.system import runner as r
from pygemstones.util import log as l

import modules.config as c
import modules.pdfium as p

# -----------------------------------------------------------------------------
def run_task_build_pdfium():
    p.get_pdfium_by_target("macos")

# -----------------------------------------------------------------------------
def run_task_patch():
    l.colored("Patching files...", l.YELLOW)

    # none

    l.ok()

# -----------------------------------------------------------------------------
def run_task_build():
    l.colored("Building libraries...", l.YELLOW)

    current_dir = f.current_dir()

    # configs
    for config in c.configurations_macos:
        # targets
        for target in c.targets_macos:
            main_dir = os.path.join(
                "build",
                target["target_os"],
                "pdfium",
                "out",
                "{0}-{1}-{2}".format(target["target_os"], target["target_cpu"], config),
            )

            f.recreate_dir(main_dir)

            os.chdir(
                os.path.join(
                    "build",
                    target["target_os"],
                    "pdfium",
                )
            )

            # generating files...
            l.colored(
                'Generating files to arch "{0}" and configuration "{1}"...'.format(
                    target["target_cpu"], config
                ),
                l.YELLOW,
            )

            arg_is_debug = "true" if config == "debug" else "false"

            args = []
            args.append('target_os="{0}"'.format(target["pdfium_os"]))
            args.append('target_cpu="{0}"'.format(target["target_cpu"]))
            args.append("use_goma=false")
            args.append("is_debug={0}".format(arg_is_debug))
            args.append("treat_warnings_as_errors=false")
            args.append("pdf_use_skia=false")
            args.append("pdf_use_skia_paths=false")
            args.append("pdf_enable_xfa=false")
            args.append("pdf_enable_v8=false")
            args.append("is_component_build=false")
            args.append("clang_use_chrome_plugins=false")
            args.append("pdf_is_standalone=true")
            args.append("use_xcode_clang=false")
            args.append("pdf_is_complete_lib=true")
            args.append("use_custom_libcxx=false")
            args.append("use_sysroot=false")

            if config == "release":
                args.append("symbol_level=0")

            args_str = " ".join(args)

            command = [
                "gn",
                "gen",
                "out/{0}-{1}-{2}".format(
                    target["target_os"], target["target_cpu"], config
                ),
                "--args='{0}'".format(args_str),
            ]
            r.run(" ".join(command), shell=True)

            # compiling...
            l.colored(
                'Compiling to arch "{0}" and configuration "{1}"...'.format(
                    target["target_cpu"], config
                ),
                l.YELLOW,
            )

            command = [
                "ninja",
                "-C",
                "out/{0}-{1}-{2}".format(
                    target["target_os"], target["target_cpu"], config
                ),
                "pdfium",
                "-v",
            ]
            r.run(command)

            os.chdir(current_dir)

    l.ok()

# -----------------------------------------------------------------------------
def run_task_install():
    l.colored("Installing libraries...", l.YELLOW)

    # configs
    for config in c.configurations_macos:
        f.recreate_dir(os.path.join("build", "macos", config))
        f.create_dir(os.path.join("build", "macos", config, "lib"))

        # targets
        for target in c.targets_macos:
            source_lib_path = os.path.join(
                "build",
                target["target_os"],
                "pdfium",
                "out",
                "{0}-{1}-{2}".format(target["target_os"], target["target_cpu"], config),
                "obj",
                "libpdfium.a",
            )

            target_lib_path = os.path.join(
                "build",
                target["target_os"],
                config,
                "lib",
                "libpdfium_{0}.a".format(target["target_cpu"]),
            )

            f.copy_file(source_lib_path, target_lib_path)

            # fix include path
            source_include_path = os.path.join(
                "build",
                target["target_os"],
                "pdfium",
                "public",
            )

            headers = f.find_files(source_include_path, "*.h", True)

            for header in headers:
                f.replace_in_file(header, '#include "public/', '#include "../')

        # universal
        folder = os.path.join("build", "macos", config, "lib", "*.a")
        files = glob.glob(folder)
        files_str = " ".join(files)
        lib_file_out = os.path.join("build", "macos", config, "lib", "libpdfium.a")

        l.colored("Merging libraries (lipo)...", l.YELLOW)
        command = ["lipo", "-create", files_str, "-o", lib_file_out]
        r.run(" ".join(command), shell=True)

        l.colored("File data...", l.YELLOW)
        command = ["file", lib_file_out]
        r.run(" ".join(command), shell=True)

        l.colored("File size...", l.YELLOW)
        command = ["ls", "-lh ", lib_file_out]
        r.run(" ".join(command), shell=True)

        # headers
        l.colored("Copying header files...", l.YELLOW)

        include_dir = os.path.join("build", "macos", "pdfium", "public")
        include_cpp_dir = os.path.join(include_dir, "cpp")
        target_include_dir = os.path.join("build", "macos", config, "include")
        target_include_cpp_dir = os.path.join(target_include_dir, "cpp")

        f.recreate_dir(target_include_dir)
        f.copy_files(include_dir, target_include_dir, "*.h")
        f.copy_files(include_cpp_dir, target_include_cpp_dir, "*.h")

    l.ok()

# -----------------------------------------------------------------------------
def run_task_test():
    l.colored("Testing...", l.YELLOW)

    current_dir = os.getcwd()
    sample_dir = os.path.join(current_dir, "sample")
    build_dir = os.path.join(sample_dir, "build")

    f.recreate_dir(build_dir)

    os.chdir(build_dir)

    # generate project
    command = ["cmake", "../"]
    r.run(command)

    # build
    command = ["cmake", "--build", "."]
    r.run(command)

    # copy assets
    f.copy_file(
        os.path.join(sample_dir, "assets", "f1.pdf"),
        os.path.join(build_dir, "f1.pdf"),
    )

    # run
    command = ["./sample"]
    r.run(command)

    # finish
    os.chdir(current_dir)

    l.ok()

# -----------------------------------------------------------------------------
def run_task_archive():
    l.colored("Archiving...", l.YELLOW)

    current_dir = f.current_dir()
    lib_dir = os.path.join(current_dir, "build", "macos")
    output_filename = os.path.join(current_dir, "macos.tgz")

    tar = tarfile.open(output_filename, "w:gz")

    for configuration in c.configurations_macos:
        tar.add(
            name=os.path.join(lib_dir, configuration),
            arcname=os.path.basename(os.path.join(lib_dir, configuration)),
            filter=lambda x: (
                None if "_" in x.name and not x.name.endswith(".h") else x
            ),
        )

    tar.close()

    l.ok()


str1=input()
y=str1
str2=input()
x=str1
for i in str2:
    if i not in x:
        str1+=i
print("Union is",end=" ")
print(str1)
z=y
res=""
for i in y:
    if i in str2 and not i in res:
        res+=i
print("Intersection is ",end=" ")
print(res)


import csv
from models.dish import Dish
from models.ingredient import Ingredient

# Req 3
class MenuData:
    def __init__(self, source_path: str) -> None:
        if not source_path:
            raise ValueError("source_path is required")
        self.source_path = source_path
        self.dishes = self.load_menu_data()

    def load_menu_data(self) -> set:
        plates = {}
        with open(self.source_path, "r", encoding="utf-8") as file:
            menu_data = csv.reader(file, delimiter=",", quotechar='"')
            next(menu_data)
            for row in menu_data:
                dish_name = row[0]
                price = float(row[1])
                ingredient = row[2]
                amount = int(row[3])
                if dish_name not in plates:
                    plates[dish_name] = Dish(dish_name, price)
                plates[dish_name].add_ingredient_dependency(
                    Ingredient(ingredient), amount
                )
        return set(plates.values())

if __name__ == "__main__":
    menu = MenuData("data/menu_base_data.csv")
    print(menu.dishes)


import csv

#infile = open('steps.csv','r')
#csvfile = csv.reader(infile,delimiter=',')

months = ['0','Janurary','Feburary','March','April','May','June','July','August','September','October','November','December']

outfile = open('avg_steps.csv','w')
#data_frame=
#writer = csv.writer(outfile,delimiter=',',newline='')

print('Month | Average steps\n')
outfile.write('Month, Average\n')

num=1

for num in range(1,13):

    steps = 0
    avg = 0
    days = 0

    infile = open('steps.csv','r')

    csvfile = csv.reader(infile, delimiter=',')

    next(csvfile)
    #i=1

    for record in csvfile:

        if str(num) == record[0]: #I need it to add all the steps and then divide it by the total number of days
            steps += int(record[1])
            days += record.count(record[0])

            print(steps)
            print(days)
            print(months[num])

        #else:
    avg = (int(steps)/int(days))

    print('#########################')
    print('%.0f'% avg)
            #print(type(i))

    print('#########################')

    avg = str(avg)

    outfile.write(months[num] + ', ' + avg+'\n')

    steps = int(record[1])
    days = int(record[0])
    num+=1

            #outfile.write(months[i],avg)
            #
    infile.close()
outfile.close()


# 2981 검문

import math

N = int(input())
nums = sorted([int(input()) for _ in range(N)])
temp = 0
for i in range(N-1):
    temp = math.gcd(temp, nums[i+1] - nums[i])

result = []
for j in range(1, int(temp**0.5)+1):
    if temp % j == 0:
        result.append(j)
        result.append(temp//j)
result = sorted(list(set(result)))
print(*result[1:], sep=' ')


import requests
import getgodville
from settings import CHATS, BOT_TOKEN, ZST_ID
from jump_counter import autostart_timers
from telebot import TeleBot
from datetime import datetime

now_str = datetime.now().strftime('%d.%m.%Y года')
prognoz = f'Всем доброе утро! Сегодня {now_str}\nУтренняя сводка новостей от бармена:\n {getgodville.god_prognoz()}\n'
coupon = getgodville.get_coupon()
for chat_id in CHATS:
    leaving = getgodville.god_guild(CHATS[chat_id])

    url = f'https://api.telegram.org/bot{BOT_TOKEN}/sendMessage?chat_id={chat_id}&text={prognoz + leaving + coupon}'
    resp = requests.get(url)
    print(f'{now_str}: {CHATS[chat_id]} status {resp.status_code}')
    # print(prognoz + leaving + coupon)

# ------<<<------ Перезапуск таймеров ------>>>------
chatlist = [ZST_ID]
user_timers = []
bot_token = BOT_TOKEN
bot = TeleBot(bot_token)
for chat in chatlist:
    autostart_timers(bot, chat, user_timers)


import json
import os

import numpy as np

from pybamocs.four_box_model.box_model import box_model
from pybamocs.four_box_model.box_model_args import dict_from_box_args

def run_and_save_box_run(box_args, save_path, run_name, compressed=False):
    M_n, M_upw, M_eddy, D_Low, T, S, sigma_0 = box_model(**box_args).unpack()
    os.makedirs(os.path.join(save_path, run_name), exist_ok=True)
    params = dict_from_box_args(**box_args)
    json.dump(params, open(os.path.join(save_path, run_name, 'args.json'), 'w'))
    if compressed:
        np.savez_compressed(os.path.join(save_path, run_name, 'results.npz'),
                            M_n=M_n, M_upw=M_upw, M_eddy=M_eddy, D_Low=D_Low, T=T, S=S, sigma_0=sigma_0)
    else:
        np.savez(os.path.join(save_path, run_name, 'results.npz'),
                 M_n=M_n, M_upw=M_upw, M_eddy=M_eddy, D_Low=D_Low, T=T, S=S, sigma_0=sigma_0)
    return True