"""
Adapted from https://www.kaggle.com/willkoehrsen/home-credit-default-risk-feature-tools and
https://www.kaggle.com/willkoehrsen/automated-feature-engineering-basics.
"""

import os

import featuretools as ft
import numpy as np
import pandas as pd

from __init__ import data_root
from data.utils import get_db_info, set_entity_variable_types, compute_and_save_dfs_features

db_name = 'homecreditdefaultrisk'
dp_limit = None
max_depth = 2
n_jobs = 1
chunk_size = 1000

if __name__ == '__main__':
    print('Loading data')
    data_dir = os.path.join(data_root, 'raw_data', db_name)
    db_info = get_db_info(db_name)

    app_train = pd.read_csv(os.path.join(data_dir, 'application_train.csv'), nrows=dp_limit, index_col=False)
    app_test = pd.read_csv(os.path.join(data_dir, 'application_test.csv'), nrows=dp_limit, index_col=False)
    bureau = pd.read_csv(os.path.join(data_dir, 'bureau.csv'), nrows=dp_limit, index_col=False)
    bureau_balance = pd.read_csv(os.path.join(data_dir, 'bureau_balance.csv'), nrows=dp_limit, index_col=False)
    cash = pd.read_csv(os.path.join(data_dir, 'POS_CASH_balance.csv'), nrows=dp_limit, index_col=False)
    credit = pd.read_csv(os.path.join(data_dir, 'credit_card_balance.csv'), nrows=dp_limit, index_col=False)
    previous = pd.read_csv(os.path.join(data_dir, 'previous_application.csv'), nrows=dp_limit, index_col=False)
    installments = pd.read_csv(os.path.join(data_dir, 'installments_payments.csv'), nrows=dp_limit, index_col=False)

    app_train['TARGET'] = app_train['TARGET'].astype(np.float64)
    app_test["TARGET"] = np.nan
    app = app_train.append(app_test, ignore_index=True)

    # Make entity set
    es = ft.EntitySet(id='applications')

    # Entities with a unique index
    es = es.entity_from_dataframe(entity_id='app',
                                  dataframe=app,
                                  index='SK_ID_CURR')
    es = es.entity_from_dataframe(entity_id='bureau',
                                  dataframe=bureau,
                                  index='SK_ID_BUREAU')
    es = es.entity_from_dataframe(entity_id='previous',
                                  dataframe=previous,
                                  index='SK_ID_PREV')
    # Entities that do not have a unique index
    es = es.entity_from_dataframe(entity_id='bureau_balance',
                                  dataframe=bureau_balance,
                                  make_index=True,
                                  index='bureaubalance_index')
    es = es.entity_from_dataframe(entity_id='cash',
                                  dataframe=cash,
                                  make_index=True,
                                  index='cash_index')
    es = es.entity_from_dataframe(entity_id='installments',
                                  dataframe=installments,
                                  make_index=True,
                                  index='installments_index')
    es = es.entity_from_dataframe(entity_id='credit',
                                  dataframe=credit,
                                  make_index=True,
                                  index='credit_index')

    # Define relationships (except featuretools can't handle multiple paths from a parent to a child)
    r_bureau_app = ft.Relationship(es['app']['SK_ID_CURR'], es['bureau']['SK_ID_CURR'])
    r_bureaubalance_bureau = ft.Relationship(es['bureau']['SK_ID_BUREAU'], es['bureau_balance']['SK_ID_BUREAU'])
    r_previousapp_app = ft.Relationship(es['app']['SK_ID_CURR'], es['previous']['SK_ID_CURR'])
    # r_cashbalance_app = ft.Relationship(es['app']['SK_ID_CURR'], es['cash']['SK_ID_PREV'])
    r_cashbalance_previousapp = ft.Relationship(es['previous']['SK_ID_PREV'], es['cash']['SK_ID_PREV'])
    # r_credit_app = ft.Relationship(es['app']['SK_ID_CURR'], es['credit']['SK_ID_CURR'])
    r_credit_previousapp = ft.Relationship(es['previous']['SK_ID_PREV'], es['credit']['SK_ID_PREV'])
    # r_installments_app = ft.Relationship(es['app']['SK_ID_CURR'], es['installments']['SK_ID_CURR'])
    r_installments_previousapp = ft.Relationship(es['previous']['SK_ID_PREV'], es['installments']['SK_ID_PREV'])

    # Add in the defined relationships
    es = es.add_relationships([r_bureau_app,
                               r_bureaubalance_bureau,
                               r_previousapp_app,
                               # r_cashbalance_app,
                               r_cashbalance_previousapp,
                               # r_credit_app,
                               r_credit_previousapp,
                               # r_installments_app,
                               r_installments_previousapp])

    # Fix up variable types
    set_entity_variable_types(es['app'], 'Application', db_info)
    set_entity_variable_types(es['bureau'], 'Bureau', db_info)
    set_entity_variable_types(es['previous'], 'PreviousApplication', db_info)
    set_entity_variable_types(es['bureau_balance'], 'BureauBalance', db_info)
    set_entity_variable_types(es['cash'], 'CashBalance', db_info)
    set_entity_variable_types(es['installments'], 'InstallmentPayment', db_info)
    set_entity_variable_types(es['credit'], 'CreditBalance', db_info)

    compute_and_save_dfs_features(es, 'app', db_name, max_depth, n_jobs, chunk_size)


import hashlib
import json
import yaml
import os
import sys
import shutil
import subprocess
import tempfile
import logging
import posixpath
import docker
import platform
from os.path import expanduser
from os.path import sep as separator
import requests
from requests.exceptions import HTTPError
import boto3
from urllib.parse import urlparse
import configparser
import base64
import uuid

from mlflow.projects.backend.abstract_backend import AbstractBackend
import mlflow.tracking as tracking
from mlflow.entities import RunStatus
from mlflow.utils.git_utils import get_git_commit, get_git_repo_url
from mlflow.projects.submitted_run import SubmittedRun
from mlflow.projects.utils import (
    fetch_and_validate_project,
    get_or_create_run,
    load_project,
    get_entry_point_command,
    get_run_env_vars,
    MLFLOW_DOCKER_WORKDIR_PATH
)
import mlflow.projects
import mlflow.projects.docker
from mlflow.utils import process, file_utils
from mlflow.utils.mlflow_tags import (
        MLFLOW_PROJECT_ENV,
        MLFLOW_PROJECT_BACKEND,
        MLFLOW_DOCKER_IMAGE_URI,
        MLFLOW_DOCKER_IMAGE_ID
)
import mlflow.projects.kubernetes
from mlflow.projects.kubernetes import KubernetesSubmittedRun, _get_run_command, _load_kube_context
import kubernetes
from kubernetes.config.config_exception import ConfigException
from parallels_plugin.login import get_conf, get_token, get_token_file_obj, get_env_var
import kubernetes
from kubernetes.config.config_exception import ConfigException

_logger = logging.getLogger(__name__)

verbose = True

'''
For running in k8s, invoke as 'mlflow run . -b infinstor-backend --backend-config kubernetes_config.json'
kubernetes_config.json contains:
{
    "backend-type": "singlevm|eks",
    "kube-context": "minikube",
    "kube-job-template-path": "kubernetes_job_template.yaml",
    "kube-client-location": "local|backend",
    "repository-uri": "public.ecr.aws/l9n7x1v8/mlflow-projects-demo/full-image",
    "kube-namespace": "default",
    "resources.limits.cpu": "500m",
    "resources.limits.memory": "512Mi",
    "resources.limits.hugepages": "2Mi/80Mi",
    "resources.limits.nvidia.com/gpu": "4",
    "resources.requests.cpu": "250m",
    "resources.requests.memory": "256Mi",
    "resources.requests.hugepages": "2Mi/80Mi"
    "resources.requests.nvidia.com/gpu": "1"
}
Notes:
- If backend-type is not present, it is assumed to be singlevm
- repository-uri is the uri where the full container img, i.e. docker env for MLproject plus MLproject files will be pushed. Use only when kube-client-location is local
- docker env for the MLproject is pushed to by: cd to the MLproject and running
$ docker build .
$ docker tag <image_id_printed_from_prev_cmd> public.ecr.aws/l9n7x1v8/mlflow-projects-demo/base-image
$ docker push public.ecr.aws/l9n7x1v8/mlflow-projects-demo/base-image

kubernetes_job_template contains:
apiVersion: batch/v1
kind: Job
metadata:
  name: "{replaced with MLflow Project name}"
  namespace: default
spec:
  ttlSecondsAfterFinished: 100
  backoffLimit: 0
  template:
    spec:
      containers:
      - name: "{replaced with MLflow Project name}"
        image: "{replaced with URI of Docker image created during Project execution}"
        command: ["{replaced with MLflow Project entry point command}"]
        #env: ["{appended with MLFLOW_TRACKING_URI, MLFLOW_RUN_ID and MLFLOW_EXPERIMENT_ID}"]
        env: []
        resources:
          limits:
            memory: 512Mi
          requests:
            memory: 256Mi
      imagePullSecrets:
      - name: ecr-private-key
      restartPolicy: Never

You can specify the kubernetes_job_template.yaml file or the k8s.resources. If you specify the job template file, and if you are using ecr private registry, you must include the imagePullSecrets section.
'''

class ParallelsSubmittedRun(SubmittedRun):
    """
    A run that just does nothing
    """

    def __init__(self, run_id):
        self._run_id = run_id
        self.status = RunStatus.RUNNING

    def wait(self):
        return True

    def get_status(self):
        return self.status;

    def set_status(self, st):
        self.status = st;

    def cancel(self):
        pass

    @property
    def run_id(self):
        return self._run_id

def upload_objects(run_id, bucket_name, path_in_bucket, local_path):
    if (path_in_bucket[0] == '/'):
        path_in_bucket = path_in_bucket[1:]
    if (verbose):
        _logger.info('upload_objects: Entered. bucket=' + bucket_name
                + ', path_in_bucket=' + path_in_bucket + ', local_path=' + local_path)
    try:
        for path, subdirs, files in os.walk(local_path):
            path = path.replace("\\","/")
            directory_name = path.replace(local_path, "")
            if directory_name.startswith('/.git'): # skip .git and subdirs
                continue
            for onefile in files:
                src_path = os.path.join(path, onefile)
                if (path_in_bucket.endswith('/')):
                    path_in_bucket = path_in_bucket[:-1]
                if (directory_name.startswith('/')):
                    directory_name = directory_name[1:]
                if (len(directory_name) > 0):
                    dst_path = path_in_bucket + '/' + directory_name
                else:
                    dst_path = path_in_bucket
                    dst_path = dst_path.rstrip('\n')
                if (verbose):
                    _logger.info('upload_objects: Uploading ' + src_path + ' to ' + dst_path)
                tracking.MlflowClient().log_artifact(run_id, src_path, dst_path)
    except Exception as err:
        _logger.info(str(err))

class PluginParallelsProjectBackend(AbstractBackend):
    def run(self, project_uri, entry_point, params,
            version, backend_config, tracking_store_uri, experiment_id):

        if (verbose):
            _logger.info("PluginParallelsProjectBackend: Entered. project_uri=" + str(project_uri)\
                + ", entry_point=" + str(entry_point)\
                + ", params=" + str(params)\
                + ", version=" + str(version)\
                + ", backend_config=" + str(backend_config)\
                + ", experiment_id=" + str(experiment_id)\
                + ", tracking_store_uri=" + str(tracking_store_uri))

        work_dir = fetch_and_validate_project(project_uri, version, entry_point, params)
        if 'run-id' in backend_config:
            active_run = get_or_create_run(backend_config['run-id'], project_uri,
                    experiment_id, work_dir, version, entry_point, params)
        else:
            active_run = get_or_create_run(None, project_uri, experiment_id, work_dir, version,
                                       entry_point, params)
        if (verbose):
            _logger.info('active_run=' + str(active_run))
            _logger.info('active_run.info=' + str(active_run.info))

        artifact_uri = active_run.info.artifact_uri
        run_id = active_run.info.run_id

        # Note: this line is screen scraped/parsed in clientlib, to extract the run_id from the output of 'mlflow run -b infinstor_backend ...' command.  See clientlib::run_transform_eks(), clientlib::run_transform_singlevm()
        _logger.info('run_id=' + str(run_id))

        tags = active_run.data.tags
        if (tags['mlflow.source.type'] != 'PROJECT'):
            raise ValueError('mlflow.source_type must be PROJECT. Instead it is '\
                    + tags['mlflow.source.type'])

        if ('parent_run_id' in backend_config):
            parent_run_id = backend_config['parent_run_id']
            tracking.MlflowClient().set_tag(active_run.info.run_id,
                    'mlflow.parentRunId', parent_run_id)

        pdst = urlparse(artifact_uri)
        bucket_name = pdst.netloc
        if (pdst.path[0] == '/'):
            path_in_bucket = pdst.path[1:]
        else:
            path_in_bucket = pdst.path

        '''
        localdir = tags['mlflow.source.name']
        if ('mlflow.source.git.repoURL' in tags or
                'mlflow.gitRepoURL' in tags or
                'mlflow.source.git.commit' in tags):
            pl = urlparse(localdir)
            if (not pl.scheme == 'file'):
                raise ValueError('Cannot deal with scheme ' + pl.scheme + ' in source path')
            localdir = pl.path + separator + pl.fragment
        '''

        project = load_project(work_dir)
        tracking.MlflowClient().set_tag(active_run.info.run_id, MLFLOW_PROJECT_BACKEND, "mlflow-parallels")

        backend_type = backend_config.get('backend-type')
        if backend_type == 'eks' or backend_type == 'gke':
            return self.run_eks(run_id, backend_type, bucket_name, path_in_bucket, work_dir, project_uri, entry_point,
                    params, version, backend_config, tracking_store_uri, experiment_id, project, active_run)
        else:
            _logger.info('Error. unknown backend type ' + backend_type)
            self.fail_run(run_id)
            rv = ParallelsSubmittedRun(active_run.info.run_id)
            rv.set_status(RunStatus.FAILED)
            return rv

    def run_eks(self, run_id, backend_type, bucket_name, path_in_bucket, work_dir, project_uri, entry_point, params,
            version, backend_config, tracking_store_uri, experiment_id, project, active_run):
        kube_client_location = backend_config.get('kube-client-location', 'local')
        if kube_client_location == 'local':
            return self.run_eks_on_local(backend_type, project_uri, entry_point, params, version,
                    backend_config, tracking_store_uri, experiment_id, project, active_run, work_dir)
        elif kube_client_location == 'backend':
            return self.run_eks_on_backend(run_id, backend_type, bucket_name, path_in_bucket, work_dir, project_uri,
                    entry_point, params, version, backend_config, tracking_store_uri, experiment_id,
                    project, active_run)
        else:
            raise ValueError('kube_client_location must be either local or backend')

    def run_eks_on_backend(self, run_id, backend_type, bucket_name, path_in_bucket, work_dir, project_uri, entry_point, params,
            version, backend_config, tracking_store_uri, experiment_id, project, active_run):
        upload_objects(run_id, bucket_name, '.mlflow-parallels/project_files', work_dir)
        body = dict()
        body['backend_type'] = backend_type
        body['MLFLOW_TRACKING_URI'] = os.getenv('MLFLOW_TRACKING_URI')
        body['MLFLOW_PARALLELS_URI'] = os.getenv('MLFLOW_PARALLELS_URI')
        body['params'] = params
        body['run_id'] = run_id
        body['experiment_id'] = str(experiment_id)
        body['docker_image'] = project.docker_env.get("image")
        if not body['docker_image']:
            self.fail_run(run_id)
            raise ValueError('Error. docker image not specified in MLproject')

        if 'kube-job-template-path' in backend_config:
            _logger.info('Using kubernetes job template file ' + backend_config['kube-job-template-path'])
            body['kube_job_template_contents'] = base64.b64encode(
                    open(backend_config.get('kube-job-template-path'), "r").read().encode('utf-8')).decode('utf-8')
            with open(backend_config.get('kube-job-template-path'), "r") as yf:
                yml = yaml.safe_load(yf)
            if 'metadata' in yml and 'namespace' in yml['metadata']:
                body['namespace'] = yml['metadata']['namespace']
                _logger.info('namespace obtained from job template: ' + body['namespace'])
                if 'kube-namespace' in backend_config:
                    if body['namespace'] != backend_config['kube-namespace']:
                        _logger.info('Error. mismatch between namespace in backend configuration and job template')
                        self.fail_run(run_id)
                        rv = ParallelsSubmittedRun(active_run.info.run_id)
                        rv.set_status(RunStatus.FAILED)
                        return rv
            else:
                if 'kube-namespace' in backend_config:
                    body['namespace'] = backend_config['kube-namespace']
                    _logger.info('namespace obtained from backend configuration: ' + body['namespace'])
                else:
                    body['namespace'] = 'default'
        else:
            _logger.info('Using parameters provided in backend configuration to generate a kubernetes job template')
            if "resources.limits.cpu" in backend_config:
                body['resources.limits.cpu'] = backend_config['resources.limits.cpu']
            if "resources.limits.memory" in backend_config:
                body['resources.limits.memory'] = backend_config['resources.limits.memory']
            if "resources.limits.hugepages" in backend_config:
                body['resources.limits.hugepages'] = backend_config['resources.limits.hugepages']
            if "resources.limits.nvidia.com/gpu" in backend_config:
                body['resources.limits.nvidia.com/gpu'] = backend_config['resources.limits.nvidia.com/gpu']
            if "resources.requests.cpu" in backend_config:
                body['resources.requests.cpu'] = backend_config['resources.requests.cpu']
            if "resources.requests.memory" in backend_config:
                body['resources.requests.memory'] = backend_config['resources.requests.memory']
            if "resources.requests.hugepages" in backend_config:
                body['resources.requests.hugepages'] = backend_config['resources.requests.hugepages']
            if "resources.requests.nvidia.com/gpu" in backend_config:
                body['resources.requests.nvidia.com/gpu'] = backend_config['resources.requests.nvidia.com/gpu']
            if 'kube-namespace' in backend_config:
                body['namespace'] = backend_config['kube-namespace']
            else:
                body['namespace'] = 'default'
        kube_context = backend_config.get('kube-context')
        if kube_context:
            body['kube_context'] = kube_context
        if ('parent_run_id' in backend_config):
            body['parent_run_id'] = backend_config['parent_run_id']
        if ('last_in_chain_of_xforms' in backend_config):
            body['last_in_chain_of_xforms'] = backend_config['last_in_chain_of_xforms']
        body['docker_repo_name'] = self.create_docker_repo_name(work_dir)
        commit = get_git_commit(work_dir)
        if commit:
            _logger.info('Using git commit ' + commit)
            body['git_commit'] = commit

        cognito_client_id, _, _, _, region = get_conf()
        token = get_token(cognito_client_id, region, True)

        headers = {
                'Content-Type': 'application/x-amz-json-1.1',
                'Authorization' : 'Bearer ' + token
                }
        url = get_env_var().rstrip('/') + '/api/2.0/mlflow/parallels/run-project'

        try:
            response = requests.post(url, data=json.dumps(body), headers=headers)
            response.raise_for_status()
        except HTTPError as http_err:
            _logger.info(f'HTTP error occurred: {http_err}')
            raise
        except Exception as err:
            _logger.info(f'Other error occurred: {err}')
            raise
        else:
            return ParallelsSubmittedRun(active_run.info.run_id)

    def fail_run(self, run_id):
        tracking.MlflowClient().set_terminated(run_id, 'FAILED')

    def create_docker_repo_name(self, work_dir):
        wd = get_git_repo_url(work_dir)
        if not wd:
            _logger.info('Unable to determine git repo. Using working dir ' + os.getcwd() + ' as docker repository name')
            return os.getcwd()
        else:
            _logger.info('Using git repo ' + wd + ' as docker repository name')
            return wd

    def run_eks_on_local(self, backend_type, project_uri, entry_point, params,
            version, backend_config, tracking_store_uri, experiment_id, project, active_run, work_dir):
        kube_context = backend_config.get('kube-context')
        repository_uri = backend_config.get('repository-uri')
        git_commit = backend_config.get('git-commit')
        kube_job_template_path = backend_config.get('kube-job-template-path')
        input_data_spec = backend_config.get('INPUT_DATA_SPEC')
        if (verbose):
            _logger.info("PluginParallelsProjectBackend.run_eks_on_local: kube-context=" + str(kube_context)\
                + ", repository-uri=" + str(repository_uri)\
                + ", git-commit=" + str(git_commit)\
                + ", kube-job-template-path=" + str(kube_job_template_path)\
                + ", input_data_spec=" + str(input_data_spec))

        from mlflow.projects.docker import (
            validate_docker_env,
            validate_docker_installation
        )
        from mlflow.projects import kubernetes as kb

        tracking.MlflowClient().set_tag(active_run.info.run_id, MLFLOW_PROJECT_ENV, "docker")
        validate_docker_env(project)
        validate_docker_installation()

        kube_config = mlflow.projects._parse_kubernetes_config(backend_config)

        #If a local image has already been created/pulled, kube_config should have it
        if 'IMAGE_TAG' in backend_config and 'IMAGE_DIGEST' in backend_config:
            _logger.info('Image already available: {}, {}'
                         .format(backend_config['IMAGE_TAG'], backend_config['IMAGE_DIGEST']))
            submitted_run = self.run_eks_job(
                project.name,
                active_run,
                backend_config['IMAGE_TAG'],
                backend_config['IMAGE_DIGEST'],
                get_entry_point_command(project, entry_point, params, backend_config['STORAGE_DIR']),
                get_run_env_vars(
                    run_id=active_run.info.run_uuid, experiment_id=active_run.info.experiment_id
                ),
                input_data_spec,
                kube_config.get("kube-context", None),
                kube_config["kube-job-template"],
            )
            return submitted_run
        else:
            _logger.info('Image not available, build or pull')

        # First, try pulling the specific tagged version of the image from the repo
        do_build = True
        if git_commit:
            docker_client = docker.from_env()
            try:
                image = docker_client.images.pull(repository_uri, tag=git_commit[:7])
            except docker.errors.ImageNotFound as inf:
                _logger.info("PluginParallelsProjectBackend.run_eks_on_local: Docker img "
                        + repository_uri + ", tag=" + git_commit[:7] + " not found. Building...")
            except docker.errors.APIError as apie:
                _logger.info("PluginParallelsProjectBackend.run_eks_on_local: Error " + str(apie)
                        + " while pulling " + repository_uri + ", tag=" + git_commit[:7])
            else:
                _logger.info("PluginParallelsProjectBackend.run_eks_on_local: image=" + str(image))
                _logger.info("PluginParallelsProjectBackend.run_eks_on_local: Docker img found "
                        + repository_uri + ", tag=" + git_commit[:7] + ". Reusing...")
                image_digest = docker_client.images.get_registry_data(image.tags[0]).id
                _logger.info("PluginParallelsProjectBackend.run_eks_on_local: image_digest=" + image_digest)
                do_build = False
        if do_build:
            _logger.info("PluginParallelsProjectBackend.run_eks_on_local: Building "
                    + repository_uri)
            image = self.build_docker_image(
                work_dir=work_dir,
                repository_uri=kube_config["repository-uri"],
                base_image=project.docker_env.get("image"),
                run_id=active_run.info.run_id,
                git_commit=git_commit
            )
            image_digest = kb.push_image_to_registry(image.tags[0])

        submitted_run = self.run_eks_job(
            project.name,
            active_run,
            image.tags[0],
            image_digest,
            get_entry_point_command(project, entry_point, params, backend_config['STORAGE_DIR']),
            get_run_env_vars(
                run_id=active_run.info.run_uuid, experiment_id=active_run.info.experiment_id
            ),
            input_data_spec,
            kube_config.get("kube-context", None),
            kube_config["kube-job-template"],
        )
        return submitted_run

    def build_docker_image(self, work_dir, repository_uri, base_image, run_id, git_commit):
        """
        Build a docker image containing the project in `work_dir`, using the base image.
        """
        from mlflow.projects.docker import (
            _create_docker_build_ctx,
            _PROJECT_TAR_ARCHIVE_NAME,
            _GENERATED_DOCKERFILE_NAME
        )
        version_string = ":" + git_commit[:7] if git_commit else ""
        image_uri = repository_uri + version_string
        dockerfile = (
            "FROM {imagename}\n COPY {build_context_path}/ {workdir}\n WORKDIR {workdir}\n"
        ).format(
            imagename=base_image,
            build_context_path=_PROJECT_TAR_ARCHIVE_NAME,
            workdir=MLFLOW_DOCKER_WORKDIR_PATH,
        )
        build_ctx_path = _create_docker_build_ctx(work_dir, dockerfile)
        with open(build_ctx_path, "rb") as docker_build_ctx:
            _logger.info("=== Building docker image %s ===", image_uri)
            client = docker.from_env()
            image, _ = client.images.build(
                tag=image_uri,
                forcerm=True,
                dockerfile=posixpath.join(_PROJECT_TAR_ARCHIVE_NAME, _GENERATED_DOCKERFILE_NAME),
                fileobj=docker_build_ctx,
                custom_context=True,
                encoding="gzip",
            )
        try:
            os.remove(build_ctx_path)
        except Exception:
            _logger.info("Temporary docker context file %s was not deleted.", build_ctx_path)
        tracking.MlflowClient().set_tag(run_id, MLFLOW_DOCKER_IMAGE_URI, image_uri)
        tracking.MlflowClient().set_tag(run_id, MLFLOW_DOCKER_IMAGE_ID, image.id)
        return image

    def run_eks_job(
        self,
        project_name,
        active_run,
        image_tag,
        image_digest,
        command,
        env_vars,
        input_data_spec,
        kube_context=None,
        job_template=None
    ):
        _logger.info('run_eks_job: Entered. project_name=' + str(project_name)\
                + ', active_run=' + str(active_run) + ', image_tag=' + str(image_tag)\
                + ', image_digest=' + str(image_digest) + ', command=' + str(command)\
                + ', env_vars=' + str(env_vars) + ', input_data_spec=' + str(input_data_spec)\
                + ', kube_context=' + str(kube_context) + ', job_template=' + str(job_template))
        env_vars['MLFLOW_PARALLELS_URI'] = os.getenv('MLFLOW_PARALLELS_URI')
        job_template = mlflow.projects.kubernetes._get_kubernetes_job_definition(
            project_name, image_tag, image_digest, _get_run_command(command), env_vars, job_template
        )
        job_name = job_template["metadata"]["name"]
        job_namespace = job_template["metadata"]["namespace"]
        _load_kube_context(context=kube_context)

        core_api_instance = kubernetes.client.CoreV1Api()
        tok = base64.b64encode(get_token_file_obj('r').read().encode('utf-8')).decode('utf-8')
        token_secret_name = 'parallelstokenfile-' + str(uuid.uuid4())
        try:
            core_api_instance.delete_namespaced_secret(namespace=job_namespace, name=token_secret_name)
        except:
            pass
        sec = kubernetes.client.V1Secret()
        sec.metadata = kubernetes.client.V1ObjectMeta(name=token_secret_name, namespace=job_namespace)
        sec.type = 'Opaque'
        sec.data = {'token': tok}
        core_api_instance.create_namespaced_secret(namespace=job_namespace, body=sec)
        aws_creds = base64.b64encode(open(os.path.join(expanduser('~'), '.aws', 'credentials'), "r").read().encode('utf-8')).decode('utf-8')
        awscreds_secret_name = 'awscredsfile-' + active_run.info.run_id
        try:
            core_api_instance.delete_namespaced_secret(namespace=job_namespace, name=awscreds_secret_name)
        except:
            pass
        sec1 = kubernetes.client.V1Secret()
        sec1.metadata = kubernetes.client.V1ObjectMeta(name=awscreds_secret_name, namespace=job_namespace)
        sec1.type = 'Opaque'
        sec1.data = {'credentials': aws_creds}
        core_api_instance.create_namespaced_secret(namespace=job_namespace, body=sec1)
        _logger.info('run_eks_job: input_data_spec = ' + str(input_data_spec))
        if input_data_spec:
            input_spec_name = 'inputdataspec-' + active_run.info.run_id
            _logger.info('run_eks_job: input_spec_name = ' + input_spec_name)
            try:
                core_api_instance.delete_namespaced_secret(namespace=job_namespace, name=input_spec_name)
            except:
                pass
            sec2 = kubernetes.client.V1Secret()
            sec2.metadata = kubernetes.client.V1ObjectMeta(name=input_spec_name, namespace=job_namespace)
            sec2.type = 'Opaque'
            sec2.data = {input_spec_name: input_data_spec}
            core_api_instance.create_namespaced_secret(namespace=job_namespace, body=sec2)

        volume_mounts = [
                    kubernetes.client.V1VolumeMount(mount_path='/root/.mlflow-parallels', name='parallels-token-file'),
                    kubernetes.client.V1VolumeMount(mount_path='/root/.aws', name='aws-creds-file')
                ]
        if input_data_spec:
            volume_mounts.append(kubernetes.client.V1VolumeMount(mount_path='/root/.mlflow-parallels-data', name=input_spec_name))
        job_template["spec"]["template"]["spec"]["containers"][0]["volumeMounts"] = volume_mounts

        job_template["spec"]["template"]["spec"]["volumes"] = [
                    kubernetes.client.V1Volume(name="parallels-token-file", secret=kubernetes.client.V1SecretVolumeSource(secret_name=token_secret_name)),
                    kubernetes.client.V1Volume(name="aws-creds-file", secret=kubernetes.client.V1SecretVolumeSource(secret_name=awscreds_secret_name))
                ]
        if input_data_spec:
            job_template["spec"]["template"]["spec"]["volumes"].append(
                    kubernetes.client.V1Volume(name=input_spec_name, secret=kubernetes.client.V1SecretVolumeSource(secret_name=input_spec_name)))

        _logger.info('run_eks_job: job_template=' + str(job_template))
        api_instance = kubernetes.client.BatchV1Api()
        resp = api_instance.create_namespaced_job(namespace=job_namespace, body=job_template, pretty=True)
        _logger.info( resp.kind + " " + resp.metadata.name +" created." )
        tracking.MlflowClient().log_param(active_run.info.run_id, 'kubernetes.job_name', job_name)
        return KubernetesSubmittedRun(active_run.info.run_id, job_name, job_namespace)


import json
import re
from sly import Lexer, Parser

class ConfigLexer(Lexer):
    tokens = {NAME, NUMBER, STRING, LPAREN, RPAREN}

    ignore = ' \t\n'
    ignore_comment = r';.*'

    NAME = r'[А-Яа-яA-Za-z][А-Яа-яA-Za-z0-9-]*'
    NUMBER = r'\d+'
    STRING = r'"[^"]*"'
    LPAREN = r'\('
    RPAREN = r'\)'

    def error(self, t):
        print(f"Ошибка: Неизвестный символ '{t.value[0]}' в позиции {t.index}")
        self.index += 1  # Продолжим разбор, игнорируя недопустимый символ

class ConfigParser(Parser):
    tokens = ConfigLexer.tokens

    @_('s_exp_list')
    def program(self, p):
        return p.s_exp_list

    @_('s_exp s_exp_list')
    def s_exp_list(self, p):
        return [p.s_exp] + p.s_exp_list

    @_('s_exp')
    def s_exp_list(self, p):
        return [p.s_exp]

    @_('LPAREN s_exp_list RPAREN')
    def s_exp(self, p):
        return p.s_exp_list

    @_('data')
    def s_exp(self, p):
        return p.data

    @_('NUMBER')
    def data(self, p):
        return int(p.NUMBER)

    @_('STRING')
    def data(self, p):
        return p.STRING[1:-1]

    @_('NAME')
    def data(self, p):
        return p.NAME

if __name__ == '__main__':
    import sys
    if len(sys.argv) != 2:
        print("Использование: python config_parser.py <имя_файла>")
        sys.exit(1)

    filename = sys.argv[1]
    with open(filename, 'r', encoding='utf-8') as file:
        input_data = file.read()

    lexer = ConfigLexer()
    parser = ConfigParser()

    try:
        result = parser.parse(lexer.tokenize(input_data))

        json_data = {"groups": [], "students": []}

        current_group = None

        for item in result:
            if isinstance(item, list) and len(item) > 0 and isinstance(item[0], str) and re.match(r'^([А-Я]{4})-', item[0]):
                current_group = item[0]
                json_data["groups"].append(current_group)
            elif isinstance(item, list) and len(item) > 0 and isinstance(item[0], str) and item[0] == "student":
                student = {
                    "name": None,
                    "group": current_group,
                    "age": None
                }
                for attr in item[1:]:
                    if isinstance(attr, list) and len(attr) == 2:
                        key = attr[0]
                        value = attr[1]
                        if key == "name":
                            student["name"] = value
                        elif key == "age":
                            student["age"] = int(value)
                        elif key == "group":
                            student["group"] = value  # Обновляем значение группы
                if student["name"] is not None and student["age"] is not None:
                    json_data["students"].append(student)

        json_result = json.dumps(json_data, indent=2, ensure_ascii=False)
        print(json_result)
    except Exception as e:
        print(f"Ошибка разбора: {e}")


RATE = 'rate'
AMOUNT_CHANGED = 'amount_changed_coin'
USD = 'usd'
DATE_CREATED = 'date_created'
TOTAL_VALUE = 'total_value_dollar'
COIN_BALANCE = 'coin_balance'
NET_CHANGE = 'net_change_dollar'
DATE = 'date'
NET_CHANGE = 'net_change_percent'
NAME = 'name'
BALANCE_CHANGE = 'balance_change_now_usd'
NATIVE_BALANCE = 'native_balance'
TRANSACTION_AMOUNT = 'amount'
AMOUNT = 'amount'
CRYPTO_BALANCE = 'balance'
ACCUMULATED_COINS = 'accumulated_coins'
ACCUMULATED_USD = 'accumulated_usd_value'
ACCOUNT_ACCUMULATED_USD = 'cummulative_account_usd'
ACTUAL_PROFIT = 'actual_profit'
TOTAL_VALUE_USD_AT_TIME = 'total_value_usd_at_time'
AMOUNT_CHANGED_VALUE_NOW = 'changed_value_now'
CHANGED_AMOUNT_DIFFERENCE_PERCENT = 'amount_changed_diff_percent'
CHANGED_AMOUNT_DIFFERENCE_USD =  'amount_changed_diff_usd'
VALUE = 'value'
PROFIT_OR_LOSS='profit_or_loss'
TYPE = 'type'

PURCHASED_RECORDS = 'purchased_records'
RECORDS = 'records'
SOLD_RECORDS = 'sold_records'

IGNORE = ['USD', 'XRP']
# ignore for testing purpose
# IGNORE = ['USD', 'ALGO', 'YFI' , 'CGLD', 'COMP' , 'MKR' , 'REN' , 'BTC']


# !/usr/bin/env Python3
# -*- coding: utf-8 -*-
# @FILE     : test_data.py
# @Author   : Pluto.
# @Time     : 2020/9/26 16:42
from selenium import webdriver

class TestData():
    def test_data(self):
        driver=webdriver.Chrome()
        driver.get("https://home.testing-studio.com")
        print(driver.execute_script("return JSON.stringify(window.performance.timing)"))


# -*- coding:utf-8 -*-
from django.views.generic import View
from django.views.decorators.csrf import csrf_exempt
from django.utils.decorators import method_decorator

from utils.django.api import api_view
from utils.decorators import mock

# Create your views here.

@api_view(methods=('GET',))
@mock(key='mock1', file='mock_data.json', ttl=3600)
def mock1(request):
    """
    @api {GET} /v1/mockapi/mock1 测试数据接口
    @apiVersion 1.0.0
    @apiName DataMock1
    @apiGroup Mock

    @apiDescription Mock 数据接口示例。

Mock 数据接口主要用来在后端 API 实现完成之前，可以快速给到前端数据，
通过使用支持注释的 JSON 文件作为 Mock 数据定义，接口定义也可以跟 Mock 数据写在一起，
方便前后端协作开发和开发维护。

前端和后端开发都可以已自己熟悉的方式编写 API 文档和 Mock 数据，并且只需要写一次。
写在 JSON 文件中的 Mock 数据，除了可以使用 `mock` 装饰器输出之外，在开发启动初期，
也可以直接由前端借助 JSMin `import` 到代码中使用。

希望可以给前后端的协作开发带来如丝般的顺滑体验，你懂的。

使用 Python 的 Docstring 编写文档，这种段落文本的缩进问题可能是会逼死强迫症患者的。
有时候觉得 Python 也不是完全热爱生命的，嗯。

    @apiSuccess {String} code 请求状态
    @apiSuccess {String} [message] 错误消息
    @apiSuccess {Object} [data] 返回数据

    """
    raise NotImplementedError()

@method_decorator([
    api_view(methods=('GET', 'POST'), login_required=False),
    csrf_exempt
], name='dispatch')
class Mock2View(View):
    """
    The APIs definition can also be written within the mock json file.
    See file "mock_data.json" for examples.

    """

    # The param file is optional, if absent, files "mock_data.json" and
    # "mock.json" will be searched
    @mock(key='mock2_get')
    def get(self, request, some_id=None):
        raise NotImplementedError()

    # You can also raise mock.PleaseMockMe to make the mock decorator
    # intercept the response.
    # When using PleaseMockMe, a different key can be specified optionally.
    @mock(key='mock2_post')
    def post(self, request):
        data = []
        if not data:
            raise mock.PleaseMockMe('mock2_post_alternative')
        return data


import os

import tvdb_v4_official

def demo1():
    print('hey')
    api_key = os.environ.get('TVDB_API_KEY')
    pin = os.environ.get('TVDB_PIN')
    print(f'api key is {api_key}')
    print(f'PIN is {pin}')
    tvdb = tvdb_v4_official.TVDB(api_key, pin)

    # fetching several pages of series info
    series_list = []
    for j in range(5):  # Pages are numbered from 0
        series_list += tvdb.get_all_series(j)

    # fetching a series given an id
    series = tvdb.get_series(121361)

    # fetching a season's episode list
    series = tvdb.get_series_extended(121361)
    for season in sorted(series['seasons'], key=lambda x: (x['type']['name'], x['number'])):
        if season['type']['name'] == 'Aired Order' and season['number'] == 1:
            season = tvdb.get_season_extended(season['id'])
            break
        else:
            season = None

    # if season is not None:
    #     for episode in season['episodes']:
    #         print(f'{episode["name"]}')

    # fetch a page of episodes from a series by season_type (type is "default" if unspecified)
    info = tvdb.get_series_episodes(121361, page=0)
    # print(info['series'])

    # for ep in info['episodes']:
    #     print(ep)

    # fetching a movie
    movie = tvdb.get_movie(31)  # avengers

    # access a movie's characters
    movie = tvdb.get_movie_extended(31)
    # for c in movie['characters']:
    #     print(c)

    # fetching a person record
    person = tvdb.get_person_extended(movie['characters'][0]['peopleId'])
    # print(person)

    # using If-Modified-Since parameter
    series = tvdb.get_series_extended(393199, if_modified_since="Wed, 30 Jun 2022 07:28:00 GMT")
    print(series)

def get_plex_name(tvdb, series_name, episode_name):
    results = tvdb.search(series_name, type='series')
    if len(results) > 1:
        print(f'{[series["name"] for series in results]}')
        result = results[1]  # hacky
    else:
        result = results[0]
    tvdb_id = int(result['tvdb_id'])

    # fetching a season's episode list
    series = tvdb.get_series_extended(tvdb_id)
    series_name = series['name']
    for series_season in sorted(series['seasons'], key=lambda x: (x['type']['name'], x['number'])):
        if series_season['type']['name'] == 'Aired Order':
            season = tvdb.get_season_extended(series_season['id'])
            for episode in season['episodes']:
                if episode['name'] == episode_name:
                    return f'{series_name} - S{series_season["number"]:02}E{episode["number"]:02} - {episode["name"]}'

def print_all_episodes(tvdb, series_name):
    results = tvdb.search(series_name, type='series')
    print(f'{len(results)} matching series')
    if len(results) > 1:
        print(f'{[series["name"] for series in results]}')
        result = results[1]  # hacky
    else:
        result = results[0]
    tvdb_id = int(result['tvdb_id'])

    # fetching a season's episode list
    series = tvdb.get_series_extended(tvdb_id)
    series_name = series['name']
    for series_season in sorted(series['seasons'], key=lambda x: (x['type']['name'], x['number'])):
        if series_season['type']['name'] == 'Aired Order':
            season = tvdb.get_season_extended(series_season['id'])
            for episode in season['episodes']:
                print(f'{series_name} - S{series_season["number"]:02}E{episode["number"]:02} - {episode["name"]}')

def demo2():
    print('demo2')
    api_key = os.environ.get('TVDB_API_KEY')
    pin = os.environ.get('TVDB_PIN')
    tvdb = tvdb_v4_official.TVDB(api_key, pin)
    # series_name = 'Foodography'
    # episode_name = 'Olive Oil'
    # episode_name = 'Los Angeles'
    # series_name = 'Build it Bigger'
    series_name = '30 Minute Meals'
    episode_names = ['Cold and New']
    for episode_name in episode_names:
        plex_name = get_plex_name(tvdb, series_name, episode_name)
        print(plex_name)
    print_all_episodes(tvdb, series_name)

if __name__ == '__main__':
    demo2()


from django.core import serializers
from django.core.serializers.json import DjangoJSONEncoder

from leads.models import WhatsappTemplate

# python manage.py shell
# from leads.whatsapp.serialize import main
# main()

# python manage.py loaddata leads/fixtures/leads_whatsapp_template.json

def main():

    data = serializers.serialize(
        'json',
        WhatsappTemplate.objects.all(),
    )

    with open('leads/fixtures/leads_whatsapp_template.json', 'w', encoding='utf-8') as out:
        out.write(data)


import argparse
from encodings import utf_8
import os
import datetime

# returns the name of the post file in the format 0n-filename.md
def slugify(s):
    if "'" in s:
        s = s.replace("'", "")
    num_of_posts_files = len(os.listdir("./posts"))
    index_to_be_added = "0" + (str(num_of_posts_files % 99))

    return index_to_be_added + "-" + "-".join(s.split(" "))

parser = argparse.ArgumentParser(description="setup blog working files and directories")

parser.add_argument(
    "-n",
    dest="new_post",
    action="store_true",
    required=True,
    help="initiate (n)ew post mode",
)
parser.add_argument("-t", dest="title", action="store", help="post (t)itle")
parser.add_argument(
    "-a", dest="arabic", action="store_true", help="creates an (a)rabic formatted file"
)
parser.add_argument("-l", dest="list", action="store_true", help="(l)ist posts")

args = parser.parse_args()

x = os.listdir(".")
if "index.html" not in x:
    html = open("index.html", "w")
    print("creating html file")
if "styles.css" not in x:
    css = open("styles.css", "w")
    print("creating styles file")
if "posts" not in x:
    os.mkdir("./posts")
    print("creating posts directory")

if args.new_post:
    if args.title:
        slug = slugify(args.title)
        if args.arabic:
            with open(f"./posts/{slug}.rtl.md", "w", encoding="utf_8") as md:
                md.write(f"u{args.title}\n\n")
                md.write(f"#programming\n\n")
                md.write(f"-" * 3 + "\n\n")
        else:
            with open(f"./posts/{slug}.md", "w") as md:
                md.write(f"{args.title}\n\n")
                md.write(f"#programming\n\n")
                md.write(f"-" * 3 + "\n\n")
        print("add new post")
    else:
        print("title needs to be added")

if args.list:
    print(os.listdir("./posts"))


import io
import os
import sys
import traceback
import datetime
from PyPDF2 import PdfFileWriter, PdfFileReader
from reportlab.pdfgen import canvas
from reportlab.lib.pagesizes import letter

def retrieve_project_parameters():

    parameters = sys.argv

    parameters_number = parameters.index("-traces") if "-traces" in parameters else None
    if parameters_number is not None:
        parameters_number = parameters_number + 1
        traces = parameters[parameters_number]
    else:
        traces = ""

    parameters_number = parameters.index("-file") if "-file" in parameters else None
    if parameters_number is not None:
        parameters_number = parameters_number + 1
        file = parameters[parameters_number]
    else:
        file = ""

    parameters_number = parameters.index("-saveas") if "-saveas" in parameters else None
    if parameters_number is not None:
        parameters_number = parameters_number + 1
        saveas = parameters[parameters_number]
    else:
        saveas = file.lower().replace(".pdf", "_new.pdf")

    parameters_number = parameters.index("-text") if "-text" in parameters else None
    if parameters_number is not None:
        parameters_number = parameters_number + 1
        text = parameters[parameters_number]
    else:
        text = ""

    parameters_number = parameters.index("-size") if "-size" in parameters else None
    if parameters_number is not None:
        parameters_number = parameters_number + 1
        size = parameters[parameters_number]
    else:
        size = ""

    parameters_number = parameters.index("-position") if "-position" in parameters else None
    if parameters_number is not None:
        parameters_number = parameters_number + 1
        position = str(parameters[parameters_number]).split(",")
    else:
        position = str(",").split(",")

    return {
        "traces": traces,
        "file": file,
        "saveas": saveas,
        "text": text,
        "size": size,
        "position": position,
    }

def validate_project_parameters(parameters):

    traces = parameters["traces"]
    file = parameters["file"]
    saveas = parameters["saveas"]
    text = parameters["text"]
    size = parameters["size"]
    position = parameters["position"]

    if traces == "" or traces.upper() == "FALSE":
        traces = False
    elif traces.upper() == "TRUE":
        traces = True
    else:
        return "ERROR: Invalid traces parameter! Parameter = " + str(traces)

    if traces is True:
        print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ": " + "=== * Parameters retrieved start * ===")

    if os.path.isfile(file):
        if not file.upper().endswith(".PDF"):
            return "ERROR: The save as file is not PDF!"
    else:
        return "ERROR: The file was not found!"

    if traces is True:
        print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ": " + "\tFile = " + str(file))

    saveas_exists = os.path.isdir(os.path.dirname(os.path.abspath(saveas)))
    if saveas_exists:
        if not saveas.upper().endswith(".PDF"):
            return "ERROR: The save as file is not PDF!"
    else:
        return "ERROR: The save as folder directory was not found!"

    if traces is True:
        print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ": " + "\tSave as = " + str(saveas))

    if traces is True:
        print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ": " + "\tText = " + str(text))

    if size != "":
        if not size.isnumeric():
            return "ERROR: The specified size is not numeric!"

    if traces is True:
        print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ": " + "\tSize = " + str(size))

    if isinstance(position, list):
        position = [i.strip() for i in position]

        for i in position:
            if not i.isnumeric():
                return "ERROR: The specified position is invalid!"

        position = {"x": position[0], "y": position[1]}

    else:
        return "ERROR: The specified position is invalid!"

    if traces is True:
        print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ": " + "\tPosition = " + str(position))

    if traces is True:
        print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ": " + "=== * Parameters retrieved end * ===")

    return {
        "traces": traces,
        "file": file,
        "saveas": saveas,
        "text": text,
        "size": size,
        "position": position,
    }

def create_template(traces, pdf_packet, size, position, text):

    if traces is True:
        print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ": " + "=== * Create template start * ===")

    can = canvas.Canvas(pdf_packet, pagesize=letter)

    if traces is True:
        print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ": " + "\tCanvas generated...")

    if size is not "" and size.isnumeric():
        size = int(size)
        can.setFont('Courier', size=size)

        if traces is True:
            print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ": " + "\tText size changed...")

    width, height = letter
    x, y = int(position["x"]), int(position["y"])
    can.drawString(width - x, height - y, text)

    if traces is True:
        print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ": " + "\tText drawn...")

    can.showPage()
    can.save()

    if traces is True:
        print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ": " + "\tTemplate saved...")

    if traces is True:
        print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ": " + "=== * Create template end * ===")

    can = None
    return pdf_packet

def generate_pdf(traces, pdf_packet, file, saveas):

    if traces is True:
        print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ": " + "=== * Generate PDF start * ===")

    pdf_packet.seek(0)
    pdf_reader = PdfFileReader(pdf_packet, strict=False)
    pdf_writer = PdfFileWriter()

    if traces is True:
        print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ": " + "\tPDF objects declared...")

    pdf_existing = PdfFileReader(open(file, "rb"), strict=False)
    pdf_existing_pages = pdf_existing.numPages

    for i in range(pdf_existing_pages):
        page = pdf_existing.getPage(i)
        page.mergePage(pdf_reader.getPage(0))
        pdf_writer.addPage(page)

        if traces is True:
            print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ": " + f"\tPage {str(i+1)} of {str(pdf_existing_pages)} handled...")

    with open(saveas, "wb") as pdf_new:
        pdf_writer.write(pdf_new)

    if traces is True:
        print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ": " + "\tOutput PDF generated...")
        print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ": " + "=== * Generate PDF end * ===")

    pdf_packet, pdf_reader, pdf_writer = None, None, None
    return pdf_packet

def execute_command(parameters):

    traces = parameters["traces"]
    file = parameters["file"]
    saveas = parameters["saveas"]
    text = parameters["text"]
    size = parameters["size"]
    position = parameters["position"]

    try:
        pdf_packet = io.BytesIO()
        pdf_packet = create_template(traces, pdf_packet, size, position, text)
        pdf_packet = generate_pdf(traces, pdf_packet, file, saveas)

    except:
        print(traceback.format_exc())
        return "ERROR: Unexpected issue!"

    return True

def main():

    parameters = retrieve_project_parameters()

    parameters = validate_project_parameters(parameters)
    if not isinstance(parameters, dict):
        print(str(parameters))
        return

    valid = execute_command(parameters)
    if not valid is True:
        print(str(valid))
        return

    print("SUCCESS")

if __name__ == "__main__":
    main()


import numpy as np
import lib.util as util

def compute_fft(data, sample_rate):
    winSampleLength, nbCh = data.shape

    # Apply Hamming window
    w = np.hamming(winSampleLength)
    dataWinCentered = data - np.mean(data, axis=0)  # Remove offset
    dataWinCenteredHam = (dataWinCentered.T * w).T

    NFFT = util.nextpow2(winSampleLength)
    Y = np.fft.fft(dataWinCenteredHam, n=NFFT, axis=0) / winSampleLength
    PSD = 2 * np.abs(Y[0:int(NFFT / 2), :])
    freq_buckets = sample_rate / 2 * np.linspace(0, 1, int(NFFT / 2))
    f = freq_buckets

    bands = {
            'delta': get_band(1, 4, f, PSD).tolist(),
            'theta': get_band(4, 8, f, PSD).tolist(),
            'alpha': get_band(8, 12, f, PSD).tolist(),
            'beta':  get_band(12, 30, f, PSD).tolist(),
            'gamma': get_band(30, 80, f, PSD).tolist(),
    }

    return PSD, freq_buckets, bands

def get_band(start, end, f, PSD):
    # Band power is set to max of all relevant frequencies
    # Other people take the sum or average
    # Max seems like the best measure IMO--no penalty for a tight peak
    return np.max(PSD[np.where((f >= start) & (f < end)), :], axis=1)[0]


"""Simple test for the Knapsack GA solver"""

import unittest
import random

from knapsack_ga_solver import *

class TestGeneratingRandomSolution(unittest.TestCase):

    def setUp(self):
        random.seed(123)
        self.items_weight_value = [
            [random.randint(1, 100), random.randint(100, 1000)] for _ in range(100)]
        self.CAPACITY = 30
        self.solution = generate_random_solution(
            self.items_weight_value, self.CAPACITY)

    def test_function_output(self):
        """Test if the function returns a list of items"""
        # Check if the solution is a list
        self.assertIsInstance(self.solution, list)
        # Check if the solution is not empty
        self.assertTrue(len(self.solution) > 0)

class TestGeneratingPopulation(TestGeneratingRandomSolution):

    def setUp(self):
        super(TestGeneratingPopulation, self).setUp()
        # Generate population
        self.population = generate_population(items=self.items_weight_value, capacity=self.CAPACITY,
                                              population_size=10, solution_generator=generate_random_solution)

    def test_function_output(self):
        """Test if the function returns a list of solutions"""
        # Check if the population is a list
        self.assertIsInstance(self.population, list)
        # Check if the population is not empty
        self.assertEqual(len(self.population), 10)
        # Check if the population is a list of lists
        for solution in self.population:
            self.assertIsInstance(solution, list)
        # Check if the population is a list of lists
        for solution in self.population:
            self.assertTrue(len(solution) > 0)
        # Check if the population is a list of lists
        for solution in self.population:
            self.assertIsInstance(solution, list)
        # Check if the population is a list of lists
        for solution in self.population:
            self.assertTrue(len(solution) > 0)
        # Check if the population is a list of lists
        for solution in self.population:
            self.assertIsInstance(solution, list)
        # Check if the population is a list of lists
        for solution in self.population:
            self.assertTrue(len(solution) > 0)

class TestUtilityCalculations(TestGeneratingRandomSolution):

    def setUp(self):
        return super().setUp()

    def test_utility_calculation(self):
        """Test if the function returns a list of solutions"""
        # Calculate utility
        utility = calculate_utility(self.solution)
        self.assertIsInstance(utility, int)
        self.assertGreater(utility, 100)

class TestGeneratingKnapsackObject(TestGeneratingRandomSolution):

    def setUp(self):
        super(TestGeneratingKnapsackObject, self).setUp()
        # Create sample knapsack object
        self.knapsack = Knapsack(population=generate_population(items=self.items_weight_value, population_size=1000,
                                                                capacity=self.CAPACITY, solution_generator=generate_random_solution))

    def test_knapsack_class(self):
        self.assertIsInstance(self.knapsack, Knapsack)

class TestGeneticFunctions(TestGeneratingKnapsackObject):

    def setUp(self):
        super(TestGeneticFunctions, self).setUp()

    def test_selection(self):
        """Test if the selection function returns a list of solutions"""
        # Select solutions
        selected_solutions = selection(self.knapsack, fitness_rank=None)
        # Check if the selected solutions is a list
        self.assertIsInstance(selected_solutions, list)
        # Check if the selected solutions is not empty
        self.assertTrue(len(selected_solutions) > 0)
        # Check if the selected solutions is a list of lists
        for solution in selected_solutions:
            self.assertIsInstance(solution, int)
        # Check if the selected solutions is a list of lists
        for solution in selected_solutions:
            self.assertIsNotNone(solution)

    def test_mutation(self):
        """Test if mutation returns a solution"""
        mutated_solution = mutation(knapsack_solution=self.knapsack, items=self.items_weight_value)
        self.assertIsInstance(mutated_solution, list)
        self.assertGreaterEqual(len(mutated_solution), 1)

    def test_crossover(self):
        """Test if crossover returns a solution"""
        crossover_solution = crossover(self.knapsack, 1, 2, 0.5)
        self.assertIsInstance(crossover_solution, list)
        self.assertGreaterEqual(len(crossover_solution), 1)


import matplotlib.pyplot as plt

from typing import List
from uxdconverter.measurement import Measurement, MeasurementContext

class Plotting(object):
    def plot(self, measurement: List[Measurement], context: MeasurementContext, names=None, counts=False):
        """
        Plots the measurement.

        If measurement is of type list, each measurement in the list is plotted in the same graph.

        :param Measurement measurement: Single measurement or a list if measurements.
        :param MeasurementContext context: Measurement context (conditions). required for plotting eg. the x-axis as
                    q (wavevector transfer) or in Theta (incidence angle)
        :param names: List of names (str) for the measurement
        :param counts: Bool, plot the reflectivity in counts (not recommended)
        :return: None
        """

        if names is not None and not len(names) == len(measurement):
            raise ValueError("Given names must have the same length as measurements")

        handles = []
        for ms in measurement:
            data = ms.get_data()
            x = [x[0] for x in data]
            x_err = [x[1] for x in data]
            y = [x[2] for x in data]
            y_err = [x[3] for x in data]

            handles.append(plt.errorbar(x, y, xerr=x_err, yerr=y_err, markeredgewidth=1, capsize=2))

        if names is not None:
            plt.legend(handles, names)

        if context.qz_conversion:
            plt.xlabel(r'$q_z$ $[A^{-1}]$')
        else:
            plt.xlabel(r'$\theta$ [deg]')

        if context.y_log_scale:
            prefix = '$log$'
        else:
            prefix = ''

        if not counts:
            suffix = "Reflectivity [1]"
        else:
            suffix = "Counts $[1]$"

        label = '{} {}'.format(prefix, suffix)

        plt.ylabel(label)

        if context.y_log_scale:
            plt.yscale('log')

        plt.show()

    def interactive_plot(self, measurement: Measurement, signal=None):
        data = measurement.get_data()

        x = [x[0] for x in data]
        y = [x[2] for x in data]
        global picked, norm
        picked = []
        norm = None

        plt.plot(x, y)
        plt.yscale('log')

        # plt.show()

        def on_click(event):
            global picked, norm
            if event.button == 1:
                if event.inaxes:
                    print('Picked coords %f %f' % (event.xdata, event.ydata))
                    picked.append((event.xdata, event.ydata))

            if event.dblclick:
                norm = [event.xdata, event.ydata]
                if not signal is None:
                    signal.sig.emit(norm)

        def on_keyboard(event):
            global picked, norm
            if event.key == 'x' and len(picked) > 0:
                norm = picked[-1]
                picked = []

        plt.gcf().canvas.mpl_connect('button_press_event', on_click)
        plt.gcf().canvas.mpl_connect('key_press_event', on_keyboard)
        plt.show()

        return norm


# -*- coding:utf-8 -*-
__author__ = 'yangjian'
"""

"""
from . import BaseEstimator
from sklearn.metrics import get_scorer

from hypergbm import HyperGBM, CompeteExperiment
from hypergbm.search_space import search_space_general, search_space_feature_gen
from hypernets.core.searcher import OptimizeDirection
from hypernets.experiment import GeneralExperiment, ConsoleCallback
from hypernets.searchers import RandomSearcher, EvolutionSearcher, MCTSSearcher
from hypernets.core import EarlyStoppingCallback, SummaryCallback

class HyperGBMEstimator(BaseEstimator):
    def __init__(self, task, scorer, reward_metric='auc', optimize_direction='max', mode='one-stage', max_trials=30, use_cache=True, earlystop_rounds=30,
                 time_limit=3600, expected_reward=None,
                 drop_feature_with_collinearity=False,
                 search_space_fn=None, ensemble_size=10, use_meta_learner=False, eval_size=0.3,
                 train_test_split_strategy=None,
                 cv=True, num_folds=3, class_balancing=None,
                 retrain_on_wholedata=False,
                 two_stage_importance_selection=True,
                 pseudo_labeling=False,
                 pseudo_labeling_proba_threshold=0.8,
                 pseudo_labeling_resplit=False,
                 **kwargs):
        super(HyperGBMEstimator, self).__init__(task)
        if kwargs.get('name') is not None:
            self.name = kwargs['name']
        else:
            self.name = 'HyperGBM'
        self.scorer = scorer
        self.reward_metric = reward_metric
        self.optimize_direction = optimize_direction
        self.mode = mode
        self.kwargs = kwargs
        self.estimator = None
        self.max_trials = max_trials
        self.use_cache = use_cache
        self.earlystop_rounds = earlystop_rounds
        self.time_limit = time_limit
        self.expected_reward = expected_reward
        self.cv = cv
        self.num_folds = num_folds
        self.search_space_fn = search_space_fn if search_space_fn is not None else lambda: search_space_general(
            early_stopping_rounds=10, verbose=0, class_balancing=class_balancing)
        self.ensemble_size = ensemble_size
        self.experiment = None
        self.use_meta_learner = use_meta_learner
        self.eval_size = eval_size
        self.train_test_split_strategy = train_test_split_strategy
        self.two_stage_importance_selection = two_stage_importance_selection
        self.retrain_on_wholedata = retrain_on_wholedata
        self.pseudo_labeling = pseudo_labeling
        self.pseudo_labeling_proba_threshold = pseudo_labeling_proba_threshold
        self.pseudo_labeling_resplit = pseudo_labeling_resplit
        self.drop_feature_with_collinearity = drop_feature_with_collinearity

    def train(self, X, y, X_test):
        # searcher = MCTSSearcher(self.search_space_fn, use_meta_learner=self.use_meta_learner, max_node_space=10,
        #                         candidates_size=10,
        #                         optimize_direction=OptimizeDirection.Maximize)
        searcher = EvolutionSearcher(self.search_space_fn,
                                     optimize_direction=self.optimize_direction, population_size=30, sample_size=10,
                                     regularized=True, candidates_size=10, use_meta_learner=self.use_meta_learner)
        # searcher = RandomSearcher(lambda: search_space_general(early_stopping_rounds=20, verbose=0),
        #                     optimize_direction=OptimizeDirection.Maximize)
        es = EarlyStoppingCallback(self.earlystop_rounds, self.optimize_direction, time_limit=self.time_limit,
                                   expected_reward=self.expected_reward)

        hk = HyperGBM(searcher, reward_metric=self.reward_metric, cache_dir=f'hypergbm_cache', clear_cache=True,
                      callbacks=[es, SummaryCallback()])

        log_callback = ConsoleCallback()
        self.experiment = CompeteExperiment(hk, X, y, X_test=X_test, eval_size=self.eval_size,
                                            train_test_split_strategy=self.train_test_split_strategy,
                                            cv=self.cv, num_folds=self.num_folds,
                                            callbacks=[],
                                            scorer=get_scorer(self.scorer),
                                            drop_feature_with_collinearity=self.drop_feature_with_collinearity,
                                            drift_detection=True,
                                            n_est_feature_importance=5,
                                            importance_threshold=1e-5,
                                            two_stage_importance_selection=self.two_stage_importance_selection,
                                            ensemble_size=self.ensemble_size,
                                            pseudo_labeling=self.pseudo_labeling,
                                            pseudo_labeling_proba_threshold=self.pseudo_labeling_proba_threshold,
                                            pseudo_labeling_resplit=self.pseudo_labeling_resplit,
                                            retrain_on_wholedata=self.retrain_on_wholedata,
                                            )
        self.estimator = self.experiment.run(use_cache=self.use_cache, max_trials=self.max_trials)

    def predict_proba(self, X):
        return self.estimator.predict_proba(X)

    def predict(self, X):
        return self.estimator.predict(X)


import logging

from dotenv import load_dotenv
from flask import Flask

load_dotenv()

from blueprints.routes import api, web
from database import restart_db_thread
from limiter import limiter
from webhook_logger import wh_logger

app = Flask(__name__)

# Register the blueprints
app.register_blueprint(web, url_prefix="/")
app.register_blueprint(api, url_prefix="/api")

@app.errorhandler(Exception)
def handle_exception(e):
    message = e.description if hasattr(e, "description") else [str(e) for a in e.args]
    response = {
        "error": {
            "type": e.__class__.__name__,
            "message": message
        }
    }

    logging.error(response)

    return response, e.code if hasattr(e, "code") else 500

if __name__ == "__main__":
    wh_logger.start()

    limiter.init_app(app)
    restart_db_thread.start()

    app.run(debug=False, host="0.0.0.0", port=1337)


# Think about everything you know about your Praxis cohort (the icebreakers doc is fair game). Think about the kind of questions that might interest you about such a corpus and the best way to structure that information to answer those questions. Write some Python code in the vein of the Scholars Lab example in the data structures review that represents the model you come up with. This one is intentionally a bit amorphous, but any practice in thinking about the shape of data is useful.

# peeps into data
caroline = {'bio': {'born':'united states', 'edu':'phd student', 'job_exp':'teaching asistant', 'dept':'art history'}, 'icebreak': {'best_tacos': 'brazos tacos', 'bad_movie':'tremors (1990)'}}
malcolm = {'bio': {'born':'united states', 'edu':'phd student', 'job_exp':'teaching asistant', 'dept':'history'}, 'icebreak': {'best_tacos': 'brazos tacos', 'bad_movie':'anything kevin hart', 'worst_dish':'iceberg lettuce'}}
samantha = {'bio': {'born':'jamaica', 'edu':'phd student', 'job_exp':['instructor', 'research assistant'], 'dept':'english'}, 'icebreak': {'best_tacos': 'don\'t know', 'bad_movie':'a knight\'s tale (2001)', 'worst_dish':'cornmeal porridge'}}
winnie = {'bio': {'born':'puerto rico', 'edu':'phd student', 'job_exp':['instructor', 'research assistant'], 'dept':'spanish'}, 'icebreak': {'best_tacos': 'don\'t know', 'bad_movie':['labyrinth (1986)', 'waiting (2005)'], 'worst_dish':'cornmeal porridge'}}

# ask questions

#1- Did the films chosen for favorite bad movie come out in a similar decade?
cohort = [caroline, malcolm, samantha, winnie]
movies = []

for person in cohort:
    str_answer = str(person['icebreak']['bad_movie'])
    movie_yrs = []

    # while loop vars
    index=0
    start=0
    end=0

    while index < len(str_answer):
        if str_answer[index] == '(':
            start = index+1
        elif str_answer[index] == ')':
            end = index
            movie_yrs.append(str_answer[start:end])
        index+=1

    for year in movie_yrs:
        print(year)

    # print(movie_yrs)
    # print('The movies came out on' + str(movie_yrs) + ', respectively.')

# ISSUES
# a. pq no sale waiting (2005)? FIXED thanks to Jeremy
# b. why can't i get nice string values out for movie_yrs? should i concatenate the lists?

# extra questions, if i ever try again
# 1-a necesitas otra variable con los años en una decada pa' compararla (la variable) con movie_yrs.
# 1-b añade regla pendeja: "si 2 < años en movie_yrs == decada, print("yup, they're from similar decades / no, they're not far apart in time")
#2- What are the worst dishes they've eaten?
#3- Is this cohort a diverse group?

# Jeremy's advice on nov/03:
# issue: pq no sale waiting (2005)?
# add elif stmt to catch whatever you're missing.
# find a way to make the start/end vars to look for the second pair of parentheses


from django.db import models
from django.core.validators import EmailValidator
import socket

class AccountType(models.Model):
    TYPES = {
        (1, 'basic'),
        (2, 'premium'),
        (3, 'enterprise')
    }
    types = models.IntegerField(default=0, choices=TYPES)

    def __str__(self):
        return f'{self.types}'

class User(models.Model):
    mail = models.CharField(unique=True, null=False, max_length=50, validators=[EmailValidator])
    account_type = models.OneToOneField(AccountType, on_delete=models.CASCADE, null=False)

    def __str__(self):
        return f'{self.mail} ({self.account_type})'

class Image(models.Model):
    user_id = models.ForeignKey(User, on_delete=models.CASCADE, related_name='images_links', null=False)
    photo = models.ImageField(null=False, blank=True)

    def __str__(self):
        return f'{self.photo} user: {self.user_id}'

    def link(self, type):
        image_link = f'http://{socket.gethostbyname("localhost")}:8000/imageuploader/users/{self.user_id.pk}/images/{self.pk}/{type}'
        return image_link

    def image_link(self):
        account_type = self.user_id.account_type.types
        small = 'small'
        medium = 'medium'
        original = 'original'
        if account_type == 1:
            return [self.link(small)]
        elif account_type == 2:
            return [self.link(small), self.link(medium)]
        else:
            return [self.link(small), self.link(medium), self.link(original)]

class Settings(models.Model):
    account_type = models.OneToOneField(AccountType, on_delete=models.CASCADE, null=False)
    resolution = models.IntegerField(null=False)

    def __str__(self):
        return f'{self.account_type} - {self.resolution}'


from rest_framework.response import Response
from . serializers import CondominiumSerializer, GroupNoticesSerializer, ApartmentSerializer
from . models import Condominium, GroupNotices,  PrivNotices, Apartment
from rest_framework import status, generics
from rest_framework.decorators import api_view
from django.contrib.auth.decorators import login_required
from rest_framework.permissions import IsAuthenticated

@api_view(['GET'])
def get_condominiums(request):
    user = request.user
    condominiums = Condominium.objects.filter(owner=user)
    serializer = CondominiumSerializer(condominiums, many = True)
    return Response(serializer.data)

@api_view(['GET'])
def get_condominiumid(request,pk):
    condominiums = Condominium.objects.get(pk=pk)
    serializer = CondominiumSerializer(condominiums)
    return Response(serializer.data)

@api_view(['POST'])
def create_condominium(request):
    serializer = CondominiumSerializer(data = request.data)
    if serializer.is_valid():
        serializer.save(owner = request.user)
        return Response(serializer.data, status=status.HTTP_201_CREATED)
    print(serializer.errors)
    return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

@api_view(['PUT'])
def edit_condominium(request,pk):
    condominium = Condominium.objects.get(pk = pk)
    if request.user == condominium.owner:
        serializer = CondominiumSerializer(condominium, data = request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status=status.HTTP_200_OK)
        return Response(serializer.data, status=status.HTTP_400_BAD_REQUEST)
    else:
        return Response(serializer.data, status=status.HTTP_401_UNAUTHORIZED)

@api_view(['DELETE'])
def delete_condominium(request, pk):
        condominium = Condominium.objects.get(pk = pk)
        if request.user == condominium.owner:
            condominium.delete()
            return Response(status = status.HTTP_204_NO_CONTENT)
        else:
            return Response(status=status.HTTP_401_UNAUTHORIZED)

@api_view(['GET'])
def get_apartments(request,pk):
    condominium = Condominium.objects.get(pk = pk)
    if request.user == condominium.owner:
        apartments = Apartment.objects.filter(condominium = condominium)
        serializer = ApartmentSerializer(apartments, many = True)
        return Response(serializer.data)
    else:
        return Response(status=status.HTTP_401_UNAUTHORIZED)

@api_view(['GET'])
def get_apartment(request,pk):
    apartment = Apartment.objects.get(pk = pk)
    serializer = ApartmentSerializer(apartment)
    return Response(serializer.data)

@api_view(['POST'])
def create_apartment(request,pk):
    request.data['renters'] = request.data.get('renters', None)
    serializer = ApartmentSerializer(data = request.data)
    condominium = Condominium.objects.get(pk = pk)
    if serializer.is_valid():
        serializer.save(condominium = condominium)
        return Response(serializer.data, status=status.HTTP_201_CREATED)
    else:
        print(serializer.errors)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

@api_view(['DELETE'])
def delete_apartment(request, pk):
        if request.user != None:
            apartment = Apartment.objects.filter(pk = pk)
            apartment.delete()
            return Response(status = status.HTTP_204_NO_CONTENT)
        else: print("ERROR")

@api_view(['PUT'])
def edit_apartment(request,pk):
    print(pk)
    apartment = Apartment.objects.get(pk = pk)
    serializer = ApartmentSerializer(apartment, data = request.data)
    if serializer.is_valid():
        serializer.save()
        return Response(serializer.data, status=status.HTTP_200_OK)
    else:
        return Response(serializer.data, status=status.HTTP_400_BAD_REQUEST)


from odoo import api, models
from odoo.http import request
from odoo.tools import ormcache

class ResUsers(models.Model):
    _inherit = "res.users"

    # Taken from oca/e-commerce/website_sale_tax_toggle
    # Start of addition
    @api.model
    def has_group(self, group_ext_id):
        tax_included = "sale.group_show_price_total"
        tax_excluded = "sale.group_show_price_subtotal"
        if (
            self.env.context.get("website_id")
            and group_ext_id in [tax_included, tax_excluded]
            and not self.env.context.get("skip_tax_toggle_check")
        ):
            taxed = request.session.get(
                "tax_toggle_taxed",
                self.env.user.with_context(
                    skip_tax_toggle_check=True
                ).has_group(tax_included),
            )
            return group_ext_id != (tax_excluded if taxed else tax_included)
        return super().has_group(group_ext_id)

    # HACK: To clear cache called from res.users write method
    @api.model
    @ormcache("self._uid", "group_ext_id")
    def _has_group(self, group_ext_id):
        return super()._has_group(group_ext_id)

    has_group.clear_cache = _has_group.clear_cache
    # End of addition


from django.utils import timezone
from django.core.urlresolvers import reverse
from django.db import models

from BearClubs.bc.models.event import Event

class Organization(models.Model):
    name                = models.CharField(max_length=128, unique=True);
    description         = models.TextField(blank=True);
    location            = models.CharField(max_length=256, blank=True);
    contact_email       = models.EmailField(max_length=128, blank=False, unique=True);
    organization_type   = models.ForeignKey('OrganizationType');
    created_at          = models.DateTimeField(default=timezone.now, editable=False);
    updated_at          = models.DateTimeField(default=timezone.now);

    @staticmethod
    def getClubsByPage(page, increment=50):
        assert(page > 0);
        assert(increment > 0);

        max_page = Organization.getMaxPage(increment);

        if page > max_page:
            page = max_page;

        # get START_INDEX based on page and increment parameters
        start_index = (page - 1) * increment;

        # only get INCREMENT items at a time
        end_index = start_index + increment;

        return Organization.objects.order_by('name')[start_index:end_index];

    @staticmethod
    def getMaxPage(increment):
        total_clubs = Organization.objects.count();
        max_page = (total_clubs // increment);

        # if there's a remainder, add one
        if total_clubs % increment != 0:
            max_page += 1;

        if max_page <= 0:
            max_page = 1;

        return max_page;

    @staticmethod
    def getEventsForOrganization(org):
        events = list()
        events_for_organization_objects = Event.objects.filter(organization=org.id);

        for event in events_for_organization_objects:
            events.append(event)

        return events

    def __str__(self):
        return self.name;

    def get_absolute_url(self):
        return reverse('club', args=[self.id]);

    def __unicode__(self):
        return 'Organization: %s' % (self.name)

    class Meta:
        app_label = 'bc';

class OrganizationType(models.Model):
    name           = models.CharField(max_length=128, unique=True);
    description    = models.CharField(max_length=256, unique=True);
    created_at     = models.DateTimeField(default=timezone.now, editable=False);
    updated_at     = models.DateTimeField(default=timezone.now);

    def __unicode__(self):
        return self.name;

    class Meta:
        app_label = 'bc';
        db_table  = 'bc_organization_type';


# 17. Double Vision - Given an array (similar to saying 'takes in an array'),
# create a function that returns a new array where each value in the original
# array has been doubled.  Calling double([1,2,3]) should return [2,4,6]
# without changing the original array

def double(arr):
    temp_arr = []
    for num in arr:
        temp_arr.append(num*2)
    return temp_arr

my_arr = [1,2,3]
print(f"Doubled array: {double(my_arr)}")
print(f"Original array: {my_arr}")


# 逆序对问题
# 在一个数组中，左边的数如果比右边的数大，则折两个数构成一个逆序对，请打印所有逆序对

"""
逆序对问题：
思路类似于小和问题
只不过是倒序排序
倒叙需要修改的是添加help_list的时机
具体首先还是两个指针a和b
a负责左半部分 b负责右半部分
当a指针指向的数大于b指针指向的数 加入a a指针向右移动
否则加入b
特别需要注意的是添加逆序对的时机和范围
首先要默认数组是排好序的 即a负责的部分与b负责的部分分别已排好序
当a大于b的时候 a指向的数与此时b指向的数后面所有的数包括b本身份别组成逆序对 加入完之后a向右移动一格
当a小于b的时候 b向右移动一格
当a再次大于b的时候 逆序对一定是以a和此时b所在位置右边所有的数包括b本身组成
因为此时b部分b所在位置之前的数一定大于a此时所在位置的数
"""

def count_inversion(x):
    if len(x) < 2:
        return 0
    return merge_sort(x, 0, len(x) - 1)

def merge_sort(x, l, r):
    if l == r:
        return []
    m = l + ((r - l) >> 1)
    return merge_sort(x, l, m) + merge_sort(x, m + 1, r) + merge(x, l, m, r)

def merge(x, l, m, r):
    p0 = l
    p1 = m + 1
    out_put = list()
    help_list = [None] * (r - l + 1)
    i = 0
    while p0 <= m and p1 <= r:
        if x[p0] < x[p1]:
            help_list[i] = x[p1]
            i += 1
            p1 += 1
        else:
            for j in range(p1, r + 1):
                out_put.append([x[p0], x[j]])
            help_list[i] = x[p0]
            i += 1
            p0 += 1
    while p0 <= m:
        help_list[i] = x[p0]
        i += 1
        p0 += 1
        # for i in range(p0, r + 1):
        #     if lists[p0] > lists[i]:
        #         out_put.append([lists[p0], lists[i]])
    while p1 <= r:
        help_list[i] = x[p1]
        i += 1
        p1 += 1

    for i in range(len(help_list)):
        x[l + i] = help_list[i]

    return out_put

if __name__ == '__main__':
    lists = [6, 4, 3, 1, 2, 5]
    result = count_inversion(lists)
    print(result)
    print(lists)


import ctypes
import numpy as np
from .dlpack import _c_str_dltensor, DLManagedTensor

class _Holder:
    """A wrapper that combines a pycapsule and array_interface for consumption by  numpy.

    Parameters
    ----------
    array_interface : dict
        A description of the underlying memory.

    pycapsule : PyCapsule
        A wrapper around the dlpack tensor that will be converted to numpy.
    """

    def __init__(self, array_interface, pycapsule) -> None:
        self.__array_interface__ = array_interface
        self._pycapsule = pycapsule

def to_numpy(pycapsule) -> np.ndarray:
    """Convert a dlpack tensor into a numpy array without copying.

    Parameters
    ----------
    pycapsule : PyCapsule
        A pycapsule wrapping a dlpack tensor that will be converted.

    Returns
    -------
    np_array : np.ndarray
        A new numpy array that uses the same underlying memory as the input
        pycapsule.
    """
    pycapsule = ctypes.py_object(pycapsule)
    assert ctypes.pythonapi.PyCapsule_IsValid(pycapsule, _c_str_dltensor)
    dl_managed_tensor = ctypes.pythonapi.PyCapsule_GetPointer(
        pycapsule, _c_str_dltensor
    )
    dl_managed_tensor = ctypes.cast(dl_managed_tensor, ctypes.POINTER(DLManagedTensor))
    holder = _Holder(dl_managed_tensor.contents.__array_interface__, pycapsule)
    return np.ctypeslib.as_array(holder)


import os
import telebot
from dotenv import load_dotenv

from Handlers.add_test_task_by_level_handler import add_easy_test_task_function, add_middle_test_task_function, \
    add_hard_test_task_function
from Handlers.add_code_task_by_level_handler import add_easy_code_task_function, add_middle_code_task_function, \
    add_hard_code_task_function
from Handlers.add_view_comments import process_comments
from change_modes import BotProcessor

from Handlers.add_admin_handler import add_admin_function
from Handlers.add_lesson_handler import add_lesson_function
from Handlers.callback_query_handler import callback_query_handler
from Handlers.check_code_handler import check_code
from Handlers.documentation_handler import search_documentation
from Handlers.add_code_task_handler import add_code_task_function
from Handlers.change_code_task_handler import change_code_task_function
from Handlers.choose_code_task_handler import process_code_task_level
from Handlers.add_test_task_handler import add_test_task_function
from Handlers.change_test_task_handler import change_test_task_function
from Handlers.choose_test_task_handler import process_test_task_level
from Handlers.choose_lesson_handler import process_lesson_topic
from Handlers.request_help_handler import help_request_handler
from Handlers.csv_handler import handle_csv_lessons
from Handlers.csv_handler import handle_csv_test_tasks
from Handlers.csv_handler import handle_csv_code_tasks
from Handlers.add_level_handler import add_level_function
from Handlers.premium_status_handler import premium_status_function
from Handlers.account_handler import account_function

from utils.key_board import InlineKeyboard, ReplyKeyboard
from utils.modes import MODE_MAIN_MENU

class Bot:
    """
    This class implements the bot
    """

    def __init__(self, token):
        self.inline_keyboard = None
        self.bot = None
        self.chat_id = None
        self.token = token

        self.current_mode = MODE_MAIN_MENU
        self.reply_keyboard = None

        self.bot_processor = BotProcessor()

    def run(self):
        """
        This method runs the bot
        """
        load_dotenv()
        token = os.getenv('TOKEN')
        # create a telebot instance using the token
        self.bot = telebot.TeleBot(token)
        # create an instance of InlineKeyboardMarkup
        self.inline_keyboard = InlineKeyboard()
        # create an instance of ReplyKeyboardMarkup
        self.reply_keyboard = ReplyKeyboard()

    def get_bot(self):
        """
        This method returns the bot
        """
        return self.bot

    def delete_message(self, chat_id, message_id):
        self.bot.delete_message(chat_id, message_id)

    def start_handler(self, message):
        """
        This method sends the welcome message
        """
        from bot import bot_db
        # print(self.bot_processor.get_current_mode())
        if not bot_db.check_user_exists(message.chat.id):
            bot_db.add_user(message.chat.id, message.chat.first_name, message.chat.username)
            self.bot.send_message(message.chat.id, f"Welcome to my bot, {message.chat.first_name}!",
                                  reply_markup=self.inline_keyboard.get_keyboard())
        else:
            self.bot.send_message(message.chat.id, f"Welcome back, {message.chat.first_name}!",
                                  reply_markup=self.inline_keyboard.get_keyboard())

    def add_lesson_handler(self, message):
        """
        This method adds a lesson
        """
        if message.text.find("/add_lesson") != -1:
            print(self.bot_processor.get_current_mode())
            add_lesson_function(self.bot, message)

    def add_admin_handler(self, message):
        """
        This method adds an admin
        """
        if message.text.find("/add_admin") != -1:
            print(self.bot_processor.get_current_mode())
            add_admin_function(self.bot, message)

    def add_test_task_handler(self, message):
        """
        This method adds a test_task
        """
        if message.text.find("/add_test_task") != -1:
            add_test_task_function(self.bot, message)

    def add_test_task_by_level_handler(self, message):
        """
        This method adds a test_task based on the level.
        """

        if message.text.find("/add_easy_test_task") != -1:
            add_easy_test_task_function(self.bot, message)
        elif message.text.find("/add_middle_test_task") != -1:
            add_middle_test_task_function(self.bot, message)
        elif message.text.find("/add_hard_test_task") != -1:
            add_hard_test_task_function(self.bot, message)

    def change_test_task_handler(self, message):
        """
        This method changes a test_task
        """
        if message.text.find("/change_test_task") != -1:
            change_test_task_function(self.bot, message)

    def add_code_task_handler(self, message):
        """
        This method adds a code_task
        """
        if message.text.find("/add_test_task") != -1:
            add_code_task_function(self.bot, message)

    def add_code_task_by_level_handler(self, message):
        """
        This method adds a code_task based on the level.
        """

        if message.text.find("/add_easy_code_task") != -1:
            add_easy_code_task_function(self.bot, message)
        elif message.text.find("/add_middle_code_task") != -1:
            add_middle_code_task_function(self.bot, message)
        elif message.text.find("/add_hard_code_task") != -1:
            add_hard_code_task_function(self.bot, message)

    def change_code_task_handler(self, message):
        """
        This method changes a test_task
        """
        if message.text.find("/change_code_task") != -1:
            change_code_task_function(self.bot, message)

    def add_level_handler(self, message):
        """
        This method adds a lesson
        """
        if message.text.find("/add_level") != -1:
            add_level_function(self.bot, message)

    def send_error_message(self, error_message):
        self.bot.send_message(os.getenv('OWNER_CHAT_ID'), f"Error in bot:\n\n{error_message}")

    def csv_tables_names_lessons(self, message):
        if message.document.file_name == 'lessons.csv':
            handle_csv_lessons(self.bot, message, message.document)
        elif message.document.file_name == 'test_tasks.csv':
            handle_csv_test_tasks(self.bot, message, message.document)
        elif message.document.file_name == 'code_tasks.csv':
            handle_csv_code_tasks(self.bot, message, message.document)
        elif message.document.file_name != 'lessons.csv' or 'test_tasks.csv':
            self.bot.send_message(message.chat.id, "Please send the file in CSV format with the same name,\n"
                                                   "as a table name in the DATABASE.")

    def handle_callback_query(self, call):

        """This handler allows using callback query with pressing designated inline keyboard buttons"""
        self.bot_processor.message_handler(call)  # Set the current mode
        callback_query_handler(call, self.bot, self.inline_keyboard, self.reply_keyboard)

    def display_current_mode(self, message):
        """
        This method displays the current mode
        """
        if self.bot_processor.is_mode_main_menu():
            self.bot.send_message(message.chat.id, "You are in the main menu mode.",
                                  reply_markup=self.inline_keyboard.get_keyboard())
        elif self.bot_processor.is_mode_documentation():
            self.bot.delete_message(message.chat.id, message.message_id)
            self.bot.send_message(message.chat.id, "You are in documentation mode.",
                                  reply_markup=self.inline_keyboard.get_keyboard())
            search_documentation(message, self.bot)
        elif self.bot_processor.is_mode_check_code():
            self.bot.send_message(message.chat.id, "You are in check code mode.",
                                  reply_markup=self.inline_keyboard.get_keyboard())
            check_code(message, self.bot)
        elif self.bot_processor.is_mode_lesson():
            self.bot.send_message(message.chat.id, "You are in lesson mode.",
                                  reply_markup=self.inline_keyboard.get_keyboard())
            process_lesson_topic(message, self.bot)
        elif self.bot_processor.is_mode_testing():
            self.bot.send_message(message.chat.id, "You are in testing mode.",
                                  reply_markup=self.inline_keyboard.get_keyboard())
            process_test_task_level(message, self.bot)
        elif self.bot_processor.is_mode_coding():
            self.bot.send_message(message.chat.id, "You are in coding mode.",
                                  reply_markup=self.inline_keyboard.get_keyboard())
            process_code_task_level(message, self.bot)
        elif self.bot_processor.is_mode_account():
            self.bot.delete_message(message.chat.id, message.message_id)
            self.bot.send_message(message.chat.id, "You are in account mode.",
                                  reply_markup=self.inline_keyboard.get_keyboard())
            account_function(message, self.bot)
        elif self.bot_processor.is_mode_comments():
            self.bot.send_message(message.chat.id, "You are in comments mode.",
                                  reply_markup=self.inline_keyboard.get_keyboard())
            process_comments(self.bot, message)
        elif self.bot_processor.is_mode_help():
            self.bot.send_message(message.chat.id, "You are in help mode.",
                                  reply_markup=self.inline_keyboard.get_keyboard())
            help_request_handler(message, self.bot)
        elif self.bot_processor.is_mode_premium():
            self.bot.send_message(message.chat.id, "You are in 'Premium' mode.",
                                  reply_markup=self.inline_keyboard.get_keyboard())
            premium_status_function(message, self.bot)

        # Handle the "HELP" button separately
        if message.text == "HELP":
            self.bot.send_message(message.chat.id, "This is the help message.",
                                  reply_markup=self.inline_keyboard.get_keyboard())