import sys

sys.path.append("..")
import imghdr
from datetime import timedelta

from werkzeug.exceptions import PreconditionFailed, InternalServerError, BadRequest
from flask import g, request
import lxml
from lxml.html.clean import Cleaner

from hackathon.database import Hackathon, User, AdminHackathonRel, DockerHostServer, HackathonLike, \
    HackathonStat, HackathonConfig, HackathonTag, UserHackathonRel, HackathonOrganizer, Award, UserHackathonAsset, \
    UserTeamRel
from hackathon.hackathon_response import internal_server_error, ok, not_found, forbidden
from hackathon.constants import HACKATHON_BASIC_INFO, ADMIN_ROLE_TYPE, HACK_STATUS, RGStatus, HTTP_HEADER, \
    FILE_TYPE, HACK_TYPE, HACKATHON_STAT, DockerHostServerStatus
from hackathon import RequiredFeature, Component, Context

docker_host_manager = RequiredFeature("docker_host_manager")
__all__ = ["HackathonManager"]

util = RequiredFeature("util")

class HackathonManager(Component):
    """Component to manage hackathon

    Note that it only handle operations directly related to Hackathon table. Things like registerd users, templates are
    in separated components
    """

    admin_manager = RequiredFeature("admin_manager")
    user_manager = RequiredFeature("user_manager")
    register_manager = RequiredFeature("register_manager")

    # basic xss prevention
    cleaner = Cleaner(safe_attrs=lxml.html.defs.safe_attrs | set(['style']))  # preserve style

    def is_hackathon_name_existed(self, name):
        """Check whether hackathon with specific name exists or not

        :type name: str|unicode
        :param name: name of hackathon

        :rtype: bool
        :return True if hackathon with specific name exists otherwise False
        """
        hackathon = self.get_hackathon_by_name(name)
        return hackathon is not None

    def is_recycle_enabled(self, hackathon):
        key = HACKATHON_BASIC_INFO.RECYCLE_ENABLED
        return self.util.str2bool(self.get_basic_property(hackathon, key, False))

    def get_hackathon_by_name(self, name):
        """Get hackathon accoring the unique name

        :type name: str|unicode
        :param name: name of hackathon

        :rtype: Hackathon
        :return hackathon instance if found else None
        """
        if not name:
            return None

        return self.db.find_first_object_by(Hackathon, name=name)

    def get_hackathon_by_id(self, hackathon_id):
        """Query hackathon by id

        :return hackathon instance or None
        """
        return self.db.find_first_object_by(Hackathon, id=hackathon_id)

    def get_hackathon_detail(self, hackathon):
        user = None
        if self.user_manager.validate_login():
            user = g.user

        return self.__get_hackathon_detail(hackathon, user)

    def get_hackathon_stat(self, hackathon):

        def internal_get_stat():
            return self.__get_hackathon_stat(hackathon)

        cache_key = "hackathon_stat_%s" % hackathon.id
        return self.cache.get_cache(key=cache_key, createfunc=internal_get_stat)

    def get_hackathon_list(self, args):
        # get values from request's QueryString
        page = int(args.get("page", 1))
        per_page = int(args.get("per_page", 20))
        order_by = args.get("order_by", "create_time")
        status = args.get("status")
        name = args.get("name")

        # build query by search conditions and order_by
        query = Hackathon.query
        if status:
            query = query.filter(Hackathon.status == status)
        if name:
            query = query.filter(Hackathon.name.like("%" + name + "%"))

        if order_by == "create_time":
            query = query.order_by(Hackathon.create_time.desc())
        elif order_by == "event_start_time":
            # all started and coming hackathon-activities would be shown based on event_start_time.
            query = query.order_by(Hackathon.event_start_time.desc())

            # just coming hackathon-activities would be shown based on event_start_time.
            # query = query.order_by(Hackathon.event_start_time.asc()).filter(Hackathon.event_start_time > self.util.get_now())
        elif order_by == "registered_users_num":
            # hackathons with zero registered users would not be shown.
            query = query.join(HackathonStat).order_by(HackathonStat.count.desc())
        else:
            query = query.order_by(Hackathon.id.desc())

        # perform db query with pagination
        pagination = self.db.paginate(query, page, per_page)

        # check whether it's anonymous user or not
        user = None
        if self.user_manager.validate_login():
            user = g.user

        def func(hackathon):
            return self.__get_hackathon_detail(hackathon, user)

        # return serializable items as well as total count
        return self.util.paginate(pagination, func)

    def get_online_hackathons(self):
        return self.db.find_all_objects(Hackathon, Hackathon.status == HACK_STATUS.ONLINE)

    def get_user_hackathon_list_with_detail(self, user_id):
        user = self.user_manager.get_user_by_id(user_id)
        user_hack_list = self.db.session().query(Hackathon, UserHackathonRel) \
            .outerjoin(UserHackathonRel, UserHackathonRel.user_id == user_id) \
            .filter(UserHackathonRel.deleted != 1, UserHackathonRel.user_id == user_id).all()

        return map(lambda h: self.__get_hackathon_detail(h, user), user_hack_list)

    def get_recyclable_hackathon_list(self):
        all_hackathon = self.db.find_all_objects(Hackathon)
        return filter(lambda h: self.is_recycle_enabled(h), all_hackathon)

    def get_entitled_hackathon_list_with_detail(self, user):
        hackathon_ids = self.admin_manager.get_entitled_hackathon_ids(user.id)
        if -1 in hackathon_ids:
            hackathon_list = self.db.find_all_objects(Hackathon)
        else:
            hackathon_list = self.db.find_all_objects(Hackathon, Hackathon.id.in_(hackathon_ids))

        return map(lambda h: self.__get_hackathon_detail(h, user), hackathon_list)

    def get_basic_property(self, hackathon, key, default=None):
        """Get basic property of hackathon from HackathonConfig"""
        config = self.db.find_first_object_by(HackathonConfig, key=key, hackathon_id=hackathon.id)
        if config:
            return config.value
        return default

    def get_all_properties(self, hackathon):
        configs = self.db.find_all_objects_by(HackathonConfig, hackathon_id=hackathon.id)
        return [c.dic() for c in configs]

    def set_basic_property(self, hackathon, properties):
        """Set basic property in table HackathonConfig"""
        if isinstance(properties, list):
            map(lambda p: self.__set_basic_property(hackathon, p), properties)
        else:
            self.__set_basic_property(hackathon, properties)

        self.cache.invalidate(self.__get_config_cache_key(hackathon))
        return ok()

    def delete_property(self, hackathon, key):
        self.db.delete_all_objects_by(HackathonConfig, hackathon_id=hackathon.id, key=key)
        return ok()

    def get_recycle_minutes(self, hackathon):
        key = HACKATHON_BASIC_INFO.RECYCLE_MINUTES
        minutes = self.get_basic_property(hackathon, key, 60)
        return int(minutes)

    def validate_hackathon_name(self):
        if HTTP_HEADER.HACKATHON_NAME in request.headers:
            try:
                hackathon_name = request.headers[HTTP_HEADER.HACKATHON_NAME]
                hackathon = self.get_hackathon_by_name(hackathon_name)
                if hackathon is None:
                    self.log.debug("cannot find hackathon by name %s" % hackathon_name)
                    return False
                else:
                    g.hackathon = hackathon
                    return True
            except Exception as ex:
                self.log.error(ex)
                self.log.debug("hackathon_name invalid")
                return False
        else:
            self.log.debug("hackathon_name not found in headers")
            return False

    def create_new_hackathon(self, context):
        """Create new hackathon based on the http body

        Hackathon name is unique so duplicated names are not allowd.

        :type context: Context
        :param context: the body of http request that contains fields to create a new hackathon

        :rtype: dict
        """
        hackathon = self.get_hackathon_by_name(context.name)
        if hackathon is not None:
            raise PreconditionFailed("hackathon name already exists")

        self.log.debug("add a new hackathon:" + context.name)
        new_hack = self.__create_hackathon(context)

        # init data is for local only
        if self.util.is_local():
            self.__create_default_data_for_local(new_hack)

        return new_hack.dic()

    def update_hackathon(self, args):
        """Update hackathon properties

        :type args: dict
        :param args: arguments from http request body that contains properties with new values

        :rtype dict
        :return hackathon in dict if updated successfully.
        """
        hackathon = g.hackathon

        try:
            update_items = self.__parse_update_items(args, hackathon)
            self.log.debug("update hackathon items :" + str(args.keys()))

            # basic xss prevention
            if 'description' in update_items and update_items['description']:
                update_items['description'] = self.cleaner.clean_html(update_items['description'])
                self.log.debug("hackathon description :" + update_items['description'])

            self.db.update_object(hackathon, **update_items)
            return hackathon.dic()
        except Exception as e:
            self.log.error(e)
            return internal_server_error("fail to update hackathon")

    def upload_files(self):
        """Handle uploaded files from http request"""
        self.__validate_upload_files()

        images = []
        storage = RequiredFeature("storage")
        for file_name in request.files:
            file_storage = request.files[file_name]
            self.log.debug("upload image file : " + file_name)
            context = Context(
                hackathon_name=g.hackathon.name,
                file_name=file_storage.filename,
                file_type=FILE_TYPE.HACK_IMAGE,
                content=file_storage
            )
            context = storage.save(context)
            image = {
                "name": file_storage.filename,
                "url": context.url,
                "thumbnailUrl": context.url,
                "deleteUrl": '/api/admin/file?key=' + context.file_name
            }
            # context.file_name is a random name created by server, file.filename is the original name
            images.append(image)

        return {"files": images}

    def like_hackathon(self, user, hackathon):
        like = self.db.find_first_object_by(HackathonLike, user_id=user.id, hackathon_id=hackathon.id)
        if not like:
            like = HackathonLike(user_id=user.id, hackathon_id=hackathon.id)
            self.db.add_object(like)
            self.db.commit()

            # increase the count of users that like this hackathon
            self.increase_hackathon_stat(hackathon, HACKATHON_STAT.LIKE, 1)

        return ok()

    def unlike_hackathon(self, user, hackathon):
        self.db.delete_all_objects_by(HackathonLike, user_id=user.id, hackathon_id=hackathon.id)
        self.db.commit()

        # sync the like count
        like_count = self.db.count_by(HackathonLike, hackathon_id=hackathon.id)
        self.update_hackathon_stat(hackathon, HACKATHON_STAT.LIKE, like_count)
        return ok()

    def update_hackathon_stat(self, hackathon, stat_type, count):
        """Increase or descrease the count for certain hackathon stat

        :type hackathon: Hackathon
        :param hackathon: instance of Hackathon to be counted

        :type stat_type: str|unicode
        :param stat_type: type of stat that defined in constants.py#HACKATHON_STAT

        :type count: int
        :param count: the new count for this stat item
        """
        stat = self.db.find_first_object_by(HackathonStat, hackathon_id=hackathon.id, type=stat_type)
        if stat:
            stat.count = count
            stat.update_time = self.util.get_now()
        else:
            stat = HackathonStat(hackathon_id=hackathon.id, type=stat_type, count=count)
            self.db.add_object(stat)

        if stat.count < 0:
            stat.count = 0
        self.db.commit()

    def increase_hackathon_stat(self, hackathon, stat_type, increase):
        """Increase or descrease the count for certain hackathon stat

        :type hackathon: Hackathon
        :param hackathon: instance of Hackathon to be counted

        :type stat_type: str|unicode
        :param stat_type: type of stat that defined in constants.py#HACKATHON_STAT

        :type increase: int
        :param increase: increase of the count. Can be positive or negative
        """
        stat = self.db.find_first_object_by(HackathonStat, hackathon_id=hackathon.id, type=stat_type)
        if stat:
            stat.count += increase
        else:
            stat = HackathonStat(hackathon_id=hackathon.id, type=stat_type, count=increase)
            self.db.add_object(stat)

        if stat.count < 0:
            stat.count = 0
        self.db.commit()

    def get_hackathon_tags(self, hackathon):
        tags = self.db.find_all_objects_by(HackathonTag, hackathon_id=hackathon.id)
        return ",".join([t.tag for t in tags])

    def set_hackathon_tags(self, hackathon, tags):
        """Set hackathon tags

        :type tags: list
        :param tags: a list of str, every str is a tag
        """
        self.db.delete_all_objects_by(HackathonTag, hackathon_id=hackathon.id)
        for tag in tags:
            t = tag.strip('"').strip("'")
            self.db.add_object(HackathonTag(tag=t, hackathon_id=hackathon.id))
        self.db.commit()
        return ok()

    def get_distinct_tags(self):
        """Return all distinct hackathon tags for auto-complete usage"""
        return self.db.session().query(HackathonTag.tag).distinct().all()

    def qet_organizer_by_id(self, organizer_id):
        organizer = self.db.get_object(HackathonOrganizer, organizer_id)
        if organizer:
            return organizer.dic()
        return not_found()

    def create_hackathon_organizer(self, hackathon, body):
        organizer = HackathonOrganizer(hackathon_id=hackathon.id,
                                       name=body["name"],
                                       organization_type=body.get("organization_type"),
                                       description=body.get("description"),
                                       homepage=body.get("homepage"),
                                       logo=body.get("logo"),
                                       create_time=self.util.get_now())
        self.db.add_object(organizer)
        return organizer.dic()

    def update_hackathon_organizer(self, hackathon, body):
        organizer = self.db.get_object(HackathonOrganizer, body["id"])
        if not organizer:
            return not_found()
        if organizer.hackathon_id != hackathon.id:
            return forbidden()

        organizer.name = body.get("name", organizer.name)
        organizer.organization_type = body.get("organization_type", organizer.organization_type)
        organizer.description = body.get("description", organizer.description)
        organizer.homepage = body.get("homepage", organizer.homepage)
        organizer.logo = body.get("logo", organizer.logo)
        organizer.update_time = self.util.get_now()
        self.db.commit()

        return organizer.dic()

    def delete_hackathon_organizer(self, hackathon, organizer_id):
        self.db.delete_all_objects_by(HackathonOrganizer, id=organizer_id, hackathon_id=hackathon.id)
        return ok()

    def create_hackathon_award(self, hackathon, body):
        level = int(body.level)
        if level > 10:
            level = 10

        award = Award(hackathon_id=hackathon.id,
                      name=body.name,
                      level=level,
                      quota=body.quota,
                      award_url=body.get("award_url"),
                      description=body.get("description"))
        self.db.add_object(award)
        return award.dic()

    def update_hackathon_award(self, hackathon, body):
        award = self.db.get_object(Award, body.id)
        if not award:
            return not_found("award not found")

        if award.hackathon.name != hackathon.name:
            return forbidden()

        level = award.level
        if body.get("level"):
            level = int(body.level)
            if level > 10:
                level = 10

        award.name = body.get("name", award.name)
        award.level = body.get("level", level)
        award.quota = body.get("quota", award.quota)
        award.award_url = body.get("award_url", award.award_url)
        award.description = body.get("description", award.description)
        award.update_time = self.util.get_now()

        self.db.commit()
        return award.dic()

    def delete_hackathon_award(self, hackathon, award_id):
        self.db.delete_all_objects_by(Award, hackathon_id=hackathon.id, id=award_id)
        return ok()

    def list_hackathon_awards(self, hackathon):
        awards = hackathon.award_contents.order_by(Award.level.desc()).all()
        return [a.dic() for a in awards]

    def schedule_pre_allocate_expr_job(self):
        """Add an interval schedule job to check all hackathons"""
        next_run_time = self.util.get_now() + timedelta(seconds=3)
        self.scheduler.add_interval(feature="hackathon_manager",
                                    method="check_hackathon_for_pre_allocate_expr",
                                    id="check_hackathon_for_pre_allocate_expr",
                                    next_run_time=next_run_time,
                                    minutes=10)

    def check_hackathon_for_pre_allocate_expr(self):
        """Check all hackathon for pre-allocate

        Add an interval job for hackathon if it's pre-allocate is enabled.
        Otherwise try to remove the schedule job
        """
        hackathon_list = self.db.find_all_objects(Hackathon)
        for hack in hackathon_list:
            job_id = "pre_allocate_expr_" + str(hack.id)
            is_job_exists = self.scheduler.has_job(job_id)
            if hack.is_pre_allocate_enabled():
                if is_job_exists:
                    self.log.debug("pre_allocate job already exists for hackathon %s" % str(hack.id))
                    continue

                self.log.debug("add pre_allocate job for hackathon %s" % str(hack.id))
                next_run_time = self.util.get_now() + timedelta(seconds=hack.id * 10)
                pre_allocate_interval = self.__get_pre_allocate_interval(hack)
                self.scheduler.add_interval(feature="expr_manager",
                                            method="pre_allocate_expr",
                                            id=job_id,
                                            context=Context(hackathon_id=hack.id),
                                            next_run_time=next_run_time,
                                            seconds=pre_allocate_interval
                                            )
            elif is_job_exists:
                self.log.debug("remove job for hackathon %s since pre_allocate is disabled" % str(hack.id))
                self.scheduler.remove_job(job_id)
        return True

    def check_hackathon_online(self, hackathon):
        alauda_enabled = is_alauda_enabled(hackathon)
        can_online = True
        if alauda_enabled == "0":
            if self.util.is_local():
                can_online = True
            else:
                can_online = docker_host_manager.check_subscription_id(hackathon.id)

        return ok(can_online)

    def __get_hackathon_detail(self, hackathon, user=None):
        """Return hackathon info as well as its details including configs, stat, organizers, like if user logon"""
        detail = hackathon.dic()

        detail["config"] = self.__get_hackathon_configs(hackathon)
        detail["stat"] = self.get_hackathon_stat(hackathon)
        detail["tag"] = self.get_hackathon_tags(hackathon)
        detail["organizer"] = self.__get_hackathon_organizers(hackathon)

        if user:
            detail["user"] = self.user_manager.user_display_info(user)
            detail["user"]["is_admin"] = self.admin_manager.is_hackathon_admin(hackathon.id, user.id)

            asset = self.db.find_all_objects_by(UserHackathonAsset, user_id=user.id, hackathon_id=hackathon.id)
            if asset:
                detail["asset"] = [o.dic() for o in asset]

            like = self.db.find_first_object_by(HackathonLike, user_id=user.id, hackathon_id=hackathon.id)
            if like:
                detail["like"] = like.dic()

            register = self.register_manager.get_registration_by_user_and_hackathon(user.id, hackathon.id)
            if register:
                detail["registration"] = register.dic()

            team_rel = self.db.find_first_object_by(UserTeamRel, user_id=user.id, hackathon_id=hackathon.id)
            if team_rel:
                detail["team"] = team_rel.team.dic()

        return detail

    def __create_hackathon(self, context):
        """Insert hackathon and admin_hackathon_rel to database

        We enforce that default config are used during the creation

        :type context: Context
        :param context: context of the args to create a new hackathon

        :rtype: Hackathon
        :return hackathon instance
        """

        new_hack = Hackathon(
            name=context.name,
            display_name=context.display_name,
            ribbon=context.get("ribbon"),
            description=context.get("description"),
            short_description=context.get("short_description"),
            banners=context.get("banners"),
            status=HACK_STATUS.INIT,
            creator_id=g.user.id,
            event_start_time=context.get("event_start_time"),
            event_end_time=context.get("event_end_time"),
            registration_start_time=context.get("registration_start_time"),
            registration_end_time=context.get("registration_end_time"),
            judge_start_time=context.get("judge_start_time"),
            judge_end_time=context.get("judge_end_time"),
            type=context.get("type", HACK_TYPE.HACKATHON)
        )

        # basic xss prevention
        if new_hack.description:  # case None type
            new_hack.description = self.cleaner.clean_html(new_hack.description)

        # insert into table hackathon
        self.db.add_object(new_hack)

        # add the current login user as admin and creator
        try:
            ahl = AdminHackathonRel(user_id=g.user.id,
                                    role_type=ADMIN_ROLE_TYPE.ADMIN,
                                    hackathon_id=new_hack.id,
                                    status=HACK_STATUS.INIT,
                                    remarks='creator',
                                    create_time=self.util.get_now())
            self.db.add_object(ahl)
        except Exception as ex:
            # TODO: send out a email to remind administrator to deal with this problems
            self.log.error(ex)
            raise InternalServerError("fail to create the default administrator")

        return new_hack

    def __get_pre_allocate_interval(self, hackathon):
        interval = self.get_basic_property(hackathon, HACKATHON_BASIC_INFO.PRE_ALLOCATE_INTERVAL_SECONDS)
        if interval:
            return int(interval)
        else:
            return 300 + hackathon.id * 10

    def __get_hackathon_configs(self, hackathon):

        def __internal_get_config():
            configs = {}
            for c in hackathon.configs.all():
                configs[c.key] = c.value
            return configs

        cache_key = self.__get_config_cache_key(hackathon)
        return self.cache.get_cache(key=cache_key, createfunc=__internal_get_config)

    def __get_hackathon_organizers(self, hackathon):
        organizers = self.db.find_all_objects_by(HackathonOrganizer, hackathon_id=hackathon.id)
        return [o.dic() for o in organizers]

    def __parse_update_items(self, args, hackathon):
        """Parse properties that need to update

        Only those whose value changed items will be returned. Also some static property like id, create_time should
        NOT be updated.

        :type args: dict
        :param args: arguments from http body which contains new values

        :type hackathon: Hackathon
        :param hackathon: the existing Hackathon object which contains old values

        :rtype: dict
        :return a dict that contains all properties that are updated.
        """
        result = {}

        for key in dict(args):
            if dict(args)[key] != hackathon.dic()[key]:
                result[key] = dict(args)[key]

        result.pop('id', None)
        result.pop('create_time', None)
        result.pop('creator_id', None)
        result['update_time'] = self.util.get_now()
        return result

    def __get_hackathon_stat(self, hackathon):
        stats = self.db.find_all_objects_by(HackathonStat, hackathon_id=hackathon.id)
        result = {
            "hackathon_id": hackathon.id,
            "online": 0,
            "offline": 0
        }
        for item in stats:
            result[item.type] = item.count

        reg_list = hackathon.registers.filter(UserHackathonRel.deleted != 1,
                                              UserHackathonRel.status.in_([RGStatus.AUTO_PASSED,
                                                                           RGStatus.AUDIT_PASSED])).all()

        reg_count = len(reg_list)
        if reg_count > 0:
            user_id_list = [r.user_id for r in reg_list]
            user_id_online = self.db.count(User, (User.id.in_(user_id_list) & (User.online == 1)))
            result["online"] = user_id_online
            result["offline"] = reg_count - user_id_online

        return result

    def __get_config_cache_key(self, hackathon):
        return "hackathon_config_%s" % hackathon.id

    def __create_default_data_for_local(self, hackathon):
        """
        create test data for new hackathon. It's for local development only
        :param hackathon:
        :return:
        """
        try:
            # test docker host server
            docker_host = DockerHostServer(vm_name="localhost", public_dns="localhost",
                                           public_ip="127.0.0.1", public_docker_api_port=4243,
                                           private_ip="127.0.0.1", private_docker_api_port=4243,
                                           container_count=0, container_max_count=100,
                                           disabled=0, state=DockerHostServerStatus.DOCKER_READY,
                                           hackathon=hackathon)
            if self.db.find_first_object_by(DockerHostServer, vm_name=docker_host.vm_name,
                                            hackathon_id=hackathon.id) is None:
                self.db.add_object(docker_host)
        except Exception as e:
            self.log.error(e)
            self.log.warn("fail to create test data")

        return

    def __validate_upload_files(self):
        # check file size
        if request.content_length > len(request.files) * self.util.get_config("storage.size_limit_kilo_bytes") * 1024:
            raise BadRequest("more than the file size limited")

        # check each file type
        for file_name in request.files:
            if request.files.get(file_name).filename.endswith('jpg'):
                continue  # jpg is not considered in imghdr
            if imghdr.what(request.files.get(file_name)) is None:
                raise BadRequest("only images can be uploaded")

    def __set_basic_property(self, hackathon, prop):
        """Set basic property in table HackathonConfig"""
        config = self.db.find_first_object_by(HackathonConfig, hackathon_id=hackathon.id, key=prop.key)
        if config:
            config.value = prop.value
        else:
            config = HackathonConfig(key=prop.key,
                                     value=prop.value,
                                     hackathon_id=hackathon.id)
            self.db.add_object(config)
        self.db.commit()

'''
Attach extension methods to Hackathon entity so that we can code like 'if hackathon.is_auto_approve(): ....' where
hackathon is entity of Hackathon that defines in database/models.py.
'''

def is_auto_approve(hackathon):
    hack_manager = RequiredFeature("hackathon_manager")
    value = hack_manager.get_basic_property(hackathon, HACKATHON_BASIC_INFO.AUTO_APPROVE, "1")
    return util.str2bool(value)

def is_pre_allocate_enabled(hackathon):
    if hackathon.status != HACK_STATUS.ONLINE:
        return False

    if hackathon.event_end_time < util.get_now():
        return False

    hack_manager = RequiredFeature("hackathon_manager")
    value = hack_manager.get_basic_property(hackathon, HACKATHON_BASIC_INFO.PRE_ALLOCATE_ENABLED, "1")
    return util.str2bool(value)

def get_pre_allocate_number(hackathon):
    hack_manager = RequiredFeature("hackathon_manager")
    value = hack_manager.get_basic_property(hackathon, HACKATHON_BASIC_INFO.PRE_ALLOCATE_NUMBER, 1)
    return int(value)

def is_alauda_enabled(hackathon):
    hack_manager = RequiredFeature("hackathon_manager")
    value = hack_manager.get_basic_property(hackathon, HACKATHON_BASIC_INFO.ALAUDA_ENABLED, "0")
    return util.str2bool(value)

def get_basic_property(hackathon, property_name, default_value=None):
    hack_manager = RequiredFeature("hackathon_manager")
    return hack_manager.get_basic_property(hackathon, property_name, default_value)

Hackathon.is_auto_approve = is_auto_approve
Hackathon.is_pre_allocate_enabled = is_pre_allocate_enabled
Hackathon.get_pre_allocate_number = get_pre_allocate_number
Hackathon.is_alauda_enabled = is_alauda_enabled
Hackathon.get_basic_property = get_basic_property


# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right
class Solution:
    def mergeTrees(self, root1: TreeNode, root2: TreeNode) -> TreeNode:

        node = TreeNode(None)

        if root1 is None and root2 is None:
            return

        if root1 and root2 is None:
            node.val = root1.val
            node.left = self.mergeTrees(root1.left, None)
            node.right = self.mergeTrees(root1.right, None)
        elif root1 is None and root2:
            node.val = root2.val
            node.left = self.mergeTrees(None, root2.left)
            node.right = self.mergeTrees(None,root2.right)
        elif root1 and root2:
            node.val = root1.val + root2.val
            node.left = self.mergeTrees(root1.left, root2.left)
            node.right = self.mergeTrees(root1.right, root2.right)

        return node


# -*- coding: utf-8 -*-

# In[] Import necessary packages
from matplotlib import pyplot as plt
import numpy as np
from PIL import Image, ImageDraw, ImageFont
import cv2

# In[]: INPUTS
#image = "selfie_002.png"
image = "pavel.jpg"
image = "rafa.jpg"

message = 'Hello! How are you? I am fine, thank you! And what about you? Do you like chocolate? Yes, sure! Wow fefeff'
message = 'Rauf is bald to you! Go to our club'
message = "Will we buy Go in such a way that there is still more and then buy more and so on until we die?"
message = "Здравствуйте всем, привет всем. Передаю привет балдёжной команде ивел панде."

# In[] LOAD IMAGE
selfie_path = "data/images/selfies/" + image
img = Image.open(selfie_path)
h,w = (384,512)
img = img.resize((h,w))
plt.imshow(img)

# In[] Binary segmentation
from deeplab_demo import DeepLabModel
seg_model_path = "deeplabv3_mnv2_pascal_train_aug_2018_01_29.tar.gz"
seg_model = DeepLabModel(seg_model_path)
resized_img, seg_map = seg_model.run(img)
resized_img = np.array(resized_img)[:w, :h]
seg_map = seg_map[:w, :h]

# In[]
plt.imshow(resized_img)

# In[]
plt.imshow(seg_map)

# In[] MORPHOLOGICAL POSTRPOCESSING OF MASK
# Largest blob
from morph import extract_largest_blob, fill_holes

mask = extract_largest_blob(seg_map)
plt.imshow(mask)

# In[]
# Filling holes
mask = fill_holes(mask)
plt.imshow(mask)

# In[] BLACK BACKGROUD ADDITION
# Add background
mask_extended = np.zeros((512, 512), dtype = np.bool)
mask_extended[-512:,:384] = mask

seg_image = resized_img * np.repeat(mask[:, :, np.newaxis], 3, axis=2)
seg_image_extended = np.zeros((512, 512, 3), dtype=np.uint8)
seg_image_extended[-512:,:384,:] = seg_image

# In[] STYLE BANK
import torch
import torchvision.transforms as transforms
import pandas as pd

import sys
sys.path.append('stylebank/')

from networks import StyleBankNet
import stylebank.util as util

content_img_transform = transforms.Compose([
	util.Resize(513),
	transforms.CenterCrop([513, 513]),
	transforms.ToTensor()
])

trans = transforms.ToPILImage()

styles = pd.read_fwf('stylebank/styles_2.txt')
styles = styles.values[:,0]

device = "cpu"
style = 'anime1'
style_idx = (styles == style).argmax()

model = StyleBankNet(1).to(device)
model.encoder_net.load_state_dict(torch.load("stylebank/weights_test_2/encoder_2.pth"))
model.decoder_net.load_state_dict(torch.load("stylebank/weights_test_2/decoder_2.pth"))
model.style_bank[0].load_state_dict(torch.load("stylebank/weights_test_2/bank_2/{}_2.pth".format(style_idx)))

x = content_img_transform(trans(seg_image_extended))
styled_image = model(x.expand((1,3,513,513)), util.get_sid_batch(list(range(1)), 1))

styled_image = styled_image[0].cpu().detach()
styled_image = styled_image.clamp(min=0, max=1)
styled_image = styled_image.cpu().numpy().transpose(1, 2, 0)
styled_image = styled_image[:w,:w,:]

#data = styled_image.astype(np.float32) / 1. # normalize the data to 0 - 1
styled_image = 255 * styled_image # Now scale by 255
styled_image = styled_image.astype(np.uint8)
plt.imshow(styled_image)

# In[] GRADIENT
def get_gradient(size_g, channel=0, sigma=1, mu=0):
    x, y = np.meshgrid(np.linspace(-1, 1, size_g), np.linspace(-1, 1, size_g))
    d = np.sqrt(x*x+y*y)
    g = np.exp(-((d-mu)**2 / (2.0 * sigma**2)))
    final = np.zeros((512, 512, 3))
    final[:, :, channel] = g
    final *= 255
    final = final.astype(np.uint8)
    return final

gradient = get_gradient(512)
plt.imshow(gradient)

# In[] TRANSPARENT OVERLAY
def transparent_overlay_2(src, overlay, pos=(0, 0), scale=1):
    overlay = cv2.resize(overlay, (0, 0), fx=scale, fy=scale)
    h, w, _ = overlay.shape  # Size of foreground
    rows, cols, _ = src.shape  # Size of background Image
    y, x = pos[0], pos[1]  # Position of foreground/overlay image

    # loop over all pixels and apply the blending equation
    for i in range(h):
        for j in range(w):
            if x + i >= rows or y + j >= cols:
                continue
            # read the alpha channel
            alpha = float(overlay[i][j][3] / 255.0)
            src[x + i][y + j] = alpha * overlay[i][j][:3] + \
                (1 - alpha) * src[x + i][y + j]

    return src

kek = styled_image*np.repeat(mask_extended[:, :, np.newaxis], 3, axis=2)
kek = cv2.cvtColor(kek, cv2.COLOR_RGB2RGBA)
kek[~mask_extended] = 0

asd = transparent_overlay_2(gradient, kek)

plt.imshow(asd)

# In[] COLOR BACKGROUD ADDITION
#tmp = cv2.cvtColor(seg_image, cv2.COLOR_RGB2GRAY)
#_, alpha = cv2.threshold(tmp,0,255,cv2.THRESH_BINARY)
#r, g, b = cv2.split(seg_image)
#rgba = [r,g,b, alpha]
#dst = cv2.merge(rgba,4)
#
#trans_mask = dst[:,:,3] == 0
#color = [0,255,0,255]
#dst[trans_mask] = color
#
#img_backgrounded = np.ones((512, 512, 4), dtype = np.uint8)*color
#img_backgrounded[-512:,:384,:] = dst
#img_backgrounded = img_backgrounded[...,:3]
#img_backgrounded = img_backgrounded.astype(np.uint8)
#
#plt.imshow(img_backgrounded)

# In[] Facial landmarks detection:
from imutils import face_utils
import dlib

detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")

gray = cv2.cvtColor(resized_img, cv2.COLOR_BGR2GRAY)

# detect faces in the grayscale image
rects = detector(gray, 1)
shape = predictor(gray, rects[0])
shape = face_utils.shape_to_np(shape)
mouth_left_xy = shape[48]
mouth_mid_xy = shape[66]
mouth_right_xy = shape[54]
mouth_x, mouth_y = mouth_right_xy
chin_xy = shape[8]

# In[] OBLACHKO:
import imageio

font_size=25
font_color=(0,0,0)
unicode_font = ImageFont.truetype("DejaVuSans.ttf", font_size)

def transparent_overlay(src, overlay, text=['Hello!'], font=cv2.FONT_HERSHEY_SIMPLEX, pos=(0, 0), pos_txt=(0, 0), scale=1):

    overlay = cv2.resize(overlay, (0, 0), fx=scale, fy=scale)
    h, w, _ = overlay.shape  # Size of foreground
    rows, cols, _ = src.shape  # Size of background Image
    y, x = pos[0], pos[1]  # Position of foreground/overlay image

    scr_cp = src.copy()
    numsteps = 3

    # loop over all pixels and apply the blending equation
    with imageio.get_writer('canadian.gif', mode='I') as writer:
        for al in range(numsteps):
            scr_cp = src.copy()
            for i in range(h):
                for j in range(w):
                    if x + i >= rows or y + j >= cols:
                        continue
                    # read the alpha channel
                    alpha = (1/numsteps)*al*float(overlay[i][j][3] / 255.0)
                    scr_cp[x + i][y + j] = alpha * overlay[i][j][:3] + \
                        (1 - alpha) * scr_cp[x + i][y + j]
            writer.append_data(scr_cp)

        canadian_img = scr_cp.copy()

        moving_mouth = canadian_img[mouth_left_xy[1]:chin_xy[1],mouth_left_xy[0]:mouth_right_xy[0],:].copy()
        mm_h, mm_w = moving_mouth.shape[:2]

        step = 10
        i = range(1, mm_h//2, step)
        j = 0

        y_pos = pos_txt[1]
        curr_img = scr_cp.copy()
        for l in text:
            curr_line = ''
            image_copy = np.zeros_like(img)
            for w in l:
                curr_line += w
                image_copy = curr_img.copy()

                image_copy[mouth_left_xy[1]:chin_xy[1],mouth_left_xy[0]:mouth_right_xy[0],:] = 0
                image_copy[mouth_left_xy[1]+i[j]:chin_xy[1]+i[j],mouth_left_xy[0]:mouth_right_xy[0],:] = moving_mouth

                #
                pilimg = Image.fromarray(image_copy)
                draw = ImageDraw.Draw(pilimg)
                draw.text ((pos_txt[0], y_pos-font_size), curr_line, font=unicode_font, fill=font_color)
                #
#                cv2.putText(image_copy, curr_line, (pos_txt[0], y_pos), font,
#                            0.8, (0, 0, 0), 1, cv2.LINE_AA)
                writer.append_data(np.array(pilimg))
                j += 1
                if (j >= len(i)):
                    j = 0
            image_copy = curr_img.copy()
            curr_img = image_copy
#            y_pos += dy

def check_text(text, max_line_length=33, max_length=100):
    lines = []
    if len(text) > max_length:
        text = text[:100]
    splitted_text = text.split()
    current_line = ''
    for word in splitted_text:
        if len(current_line+word) <= max_line_length:
            current_line = current_line + ' ' + word
        else:
            lines.append(current_line)
            current_line = word
    lines.append(current_line)
    return lines

cloud = cv2.imread('data/images/clouds/cloud4.png', -1)

cloud_w = asd.shape[0] - mouth_x
cloud_ratio = cloud.shape[0]/cloud.shape[1]
cloud_h = int(cloud_ratio*cloud_w)
cloud_resized = cv2.resize(cloud, (cloud_w, cloud_h))
cloud_pos_x = mouth_x
cloud_pos_y = mouth_y - int(cloud_h*(5/4))
cloud_center_x = cloud_pos_x + int(cloud_w/5)
lines = check_text(message, max_line_length=cloud_w//20)
cloud_center_y = cloud_pos_y + int(cloud_h/2)
transparent_overlay(asd, cloud_resized, text=lines, pos=(cloud_pos_x, cloud_pos_y), pos_txt=(cloud_center_x, cloud_center_y))


# 소수인지를 판단하는 프로그램

def prime(num) :
    for div in range(2,num) :
        if num % div == 0:
            print("소수가 아닙니다.")
            break
        elif div == num -1:
            print("소수입니다.")
        else:
            continue

number = int(input())
prime(number)


import pytorch_lightning as pl

import ray

from tablestakes.ml import model_transformer, hyperparams, metrics_mod

num_gpus = 1
num_cpus = 2

ray.init(
    address='auto',
    ignore_reinit_error=True,
)

@ray.remote(num_gpus=num_gpus, num_cpus=num_cpus, max_calls=1)
def run_one(hp: hyperparams.LearningParams):
    net = model_transformer.RectTransformerModule(hp)

    pl_callbacks = [
        metrics_mod.CounterTimerCallback(),
    ]

    trainer = pl.Trainer(
        logger=metrics_mod.get_pl_logger(hp),
        callbacks=pl_callbacks,
        max_epochs=hp.num_epochs,
        weights_summary='full',
        profiler=True,
        gpus=num_gpus,
    )

    print("Starting trainer.fit:")
    trainer.fit(net)

    print('done! with fit')

    return True

if __name__ == '__main__':
    dataset_name = 'num=10000_99e0'

    encoder_types = [
        'torch',
        # 'fast_default',
        # 'fast_favor',
        # 'fast_grf',
        # # 'performer',
        # 'ablatable_do_drop_k',
        # 'ablatable_do_not_drop_k',
    ]
    encoder_types.reverse()

    hp = hyperparams.LearningParams(dataset_name)
    hp.num_epochs = 10
    hp.lr = 0.001

    print('')
    outs = []
    for encoder_type in encoder_types:
        print(f'Starting {encoder_type}')
        hp.trans_encoder_type = encoder_type
        hp.experiment_tags = ['encoder_benchmark_v2-reverse']
        outs.append(run_one.remote(hp))

    print(ray.get(outs))
    print('done')


import openpyxl
import argparse
from efficient_apriori import apriori

parser = argparse.ArgumentParser()
parser.add_argument('text', type=str,
                help="확인 하고싶은 단어")
args = parser.parse_args()

test = openpyxl.load_workbook(filename='result.xlsx')
resultSheet = test.worksheets[0]

exception = openpyxl.load_workbook(filename='exception.xlsx')
exceptionSheet = exception.worksheets[0]

max_data_row = resultSheet.max_row + 1
max_exceoption_row = exceptionSheet.max_row + 1

for i in range(2, max_exceoption_row):
    if(exceptionSheet.cell(row = i, column = 6).value==None):
        max_exceoption_row = i
        break

data = []
for i in range(2,max_data_row):
    data.append(str(resultSheet.cell(row=i,column=1).value) + str(resultSheet.cell(row=i,column=2).value) + str(resultSheet.cell(row=i,column=3).value))

preData = []
for j in range(0,len(data)):
    temp = data[j].replace("/", " ").split(" ")
    tempData = []
    for k in range(0,len(temp)):
        if(temp[k]==''):
            continue
        temp1 = temp[k].replace("*"," ")
        tempData.append(temp1)
    tempData = list(set(tempData))
    preData.append(tuple(tempData))

itemsets, rules = apriori(preData, min_support=0.15,  min_confidence=0.1)

dd = []
rules_rhs = filter(lambda rule: len(rule.lhs) == 1 and len(rule.rhs) == 1, rules)
for rule in sorted(rules_rhs, key=lambda rule: rule.confidence):
    if(rule.lhs[0]==args.text and rule.confidence!=1 and rule.lift>1 and rule.conviction < 1.5):
        for i in range(0, len(rule.rhs)):
            dd.append(rule.rhs[i])
dd.reverse()

for i in range(0,len(dd)):
    for j in range(2,max_exceoption_row):
        if(str(exceptionSheet.cell(row=j,column=6).value).find(dd[i])!=-1):
            tempStr = str(exceptionSheet.cell(row=j,column=6).value).split("/")
            dd[i] = tempStr[0]
            break

resultData = []
for i in range(0,len(dd)):
    check = False
    for j in range(0,len(resultData)):
        if(dd[i]==resultData[j]):
            check =True
    if(check==False):
        resultData.append(dd[i])

for i in range(0,len(resultData)):
    print(resultData[i])


import sys
import argparse
import random

# parser = argparse.ArgumentParser()
# parser.add_argument("n")
# args = parser.parse_args()

n = [2**5, 2**10, 2**15, 2**20, 2**25]

for k in range(len(n)):
    print(n[k])
    with open(str(n[k])+'.txt', 'w') as fp:
        fp.write(str(n[k])+'\n')
        for i in range(n[k]):
            fp.write(str(random.uniform(-100, 100))+' ')
        fp.write('\n')
        for i in range(n[k]):
            fp.write(str(random.uniform(-100, 100))+' ')


import pyttsx3
import datetime
import speech_recognition as sr
import webbrowser
import os
import smtplib
import subprocess
import requests
import wikipedia
import ctypes
import time

engine =pyttsx3.init()
voices=engine.getProperty('voices')
engine.setProperty('voice',voices[2].id)

def speak(audio):
    engine.say(audio)
    engine.runAndWait()

def timeda():
    Time = datetime.datetime.now().strftime("%I:%M:%S")
    speak(Time)

def date():
    year = int(datetime.datetime.now().year)
    month = int(datetime.datetime.now().month)
    date = int(datetime.datetime.now().day)
    speak(date)
    speak(month)
    speak(year)

def check():
    hour = datetime.datetime.now().hour
    if hour >= 5 and hour <12:
        speak("Good Morning ")
    elif hour >=12 and hour <18:
        speak("Good Afternoon ")
    else:
        speak("Good Evening ")

def greetings():
    check()
    speak("Welcome back,Sir. Mark here")
    speak("Let me know what can i do for you sir!!")

def takeCommand():
    r = sr.Recognizer()
    with sr.Microphone() as source:
        r.pause_threshold = 1
        audio =r.listen(source)
    try:
        print("Recognising...")
        query=r.recognize_google(audio,language='en-in')
        print(query)
    except Exception as e:
        print(e)
        return "None"
    return query

def sendEmail(to, content):
    server = smtplib.SMTP('smtp.gmail.com', 587)
    server.ehlo()
    server.starttls()
    server.login('sunnykumar12928@gmail.com', '25929031')
    server.sendmail('sunnykumar12928@gmail.com', to, content)
    server.close()

def changeWallpaper():
    imgage=r"C:\Users\91800\Desktop\Mark\image\mark.jpg"
    ctypes.windll.user32.SystemParametersInfoW(20,0,imgage,0)

def changeScreen():
    app = r"C:\Users\91800\Desktop\Mark\Rainmeter\rainmeter.exe"
    os.startfile(app)
    changeWallpaper()

speak("Hello;This is Mark ,version1 point o;  Your personal , AI assistant")
speak("Its")
timeda()

while True:
    comm = takeCommand().lower()
    if "hello" in comm:
        greetings()
    elif "open youtube" in comm:
        speak("opening youtube")
        webbrowser.open("www.youtube.com")

    elif "close youtube" in comm:
        speak("closing it")
        os.system("taskkill /f /im chrome.exe")

    elif "activate" in comm:
        changeScreen()
        time.sleep(3)

    elif "open college" in comm:
        speak("opening college portal")
        webbrowser.open("https://ipec.codetantra.com/login.jsp")

    elif "offline" in comm:
        speak("Starting all application shutdown sequence")
        os.system("taskkill /f /im rainmeter.exe")
        speak("I am going offline,Sir;have a nice day")
        image=r"C:\Users\91800\Dekstop\Mark\image\windows.jpg"
        ctypes.windll.user32.SystemParametersInfoW(20,0,image,0)
        exit()

    elif "shutdown" in comm:
        speak("shutting down;sir,have a nice day")
        os.system('shutdown -s')

    elif 'open gmail' in comm:
        speak('openning gmail')
        webbrowser.open('www.gmail.com')

    elif 'search' in comm:
        speak("Searching...")
        webbrowser.open(comm)

    elif "wikipedia" in comm:
        try:
            speak("Searching on Wikipedia...")
            query = comm.replace("wikipedia", "")
            results = wikipedia.summary(query, sentences=2)
            speak("According to Wikipedia")
            speak(results)
            speak("Thank You sir ;Anything else i can do for you?")
        except Exception as e:
            speak("There is no results found for it sir ;Anything else i can do for you?")

    elif 'email to me' and 'send email' in comm:
            try:
                speak("What should I say?")
                content = takeCommand()
                speak("To whom i should send this email")
                to = takeCommand()
                sendEmail(to, content)
                speak("Email has been sent!")
                speak("Want me to open gmail")
                if "Yes" in comm:
                    webbrowser.open('www.gmail.com')
                else:
                    speak("Thank you sir ,Anthying More sir?")
            except Exception as e:
                print(e)
                speak("Sorry buddy .I am not able to send this email")

    elif "buddy" in comm:
        speak("Yes sir!!")

    elif "tell me about you" in comm:
        speak("I am Mark 1 .o created by Mister Aneesh . I have been created on A I M L and can develop with my last mistakes . Want to test me?")

    elif "play some songs" in comm:
        speak("Opening Songs")
        appli = r"C:\Users\91800\Desktop\Spotify.lnk"
        os.startfile(appli)

    elif "close it" in comm:
        os.system("taskkill /f /im Spotify.lnk")

    else:
        speak("Can you speak it again,sir")


'''# use longhand method when you need to work on each list item separately
numbers = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
for index in range(len(numbers)):
	squared = numbers[index] ** 2
	print(f'{numbers[index] :>2d} squared is {squared :>3d}')'''

print('Welcome to the Guest List Builder')

guests = []
while True:
	newGuest = name_regex()
	if newGuest == '':
		break
	else:
		guests.append(newGuest)
#print(guests) # compare results
print(*guests, sep = ", ")

for index in range(len(guests)):
	if index < len(guests) -1:
		print(f'Guest #{index + 1} is {guests[index]}, ', end = "")
	else:
		print(f'and Guest #{index + 1} is {guests[index]}.')

print('{:<30}{:>7}'.format('Guest Name','Ticket'))
for index in range(len(guests)):
	print('{:<30}{:>2}{:>5}'.format(guests[index],'#',index + 1001))
print('{:.<30}{:.>7}'.format('Total',len(guests)))

# print(guests)
# nixedSecondGuest = guests.pop(1)
# print('Name removed: ', nixedSecondGuest )
# print('Guest list: ', guests)
#
# if 'Sauron' in guests:
# 	guests.remove('Sauron')
# else:
# 	print('No such guest.')
# print('Guest list: ', guests)

# guests.sort()
# print(guests)


from base.baseachievementgame import BaseAchievementGame
from base.basegame import BaseGame
from game.model.computer import Computer
from model.region import Region
from util.globals import *

class GameD(BaseGame):
    def __init__(self):
        super().__init__()
        self.turn_cnt = 0

    def init(self):
        super().init()
        self.turn_cnt = 0
        self.deck.set_cards(self.get_deck())
        self.deal()

    def get_deck(self):
        color = list(CARD_COLOR_SET.keys())
        cards = []
        for c in color[1:]:
            for v in range(1, 10):
                cards.append(Card(c, v))
        return cards


#!/usr/bin/env python3
# ==============================================================================
# Python210 | Fall 2020
# ------------------------------------------------------------------------------
# Lesson03
# List Lab (list_lab.py)
# Steve Long 2020-09-27 | v0
#
# Requirements:
# =============
#
# In your student dir in the class repo, create a lesson03 dir and put in a new
# list_lab.py file. The file should be an executable Python script. That is to
# say that one should be able to run the script directly like so:
#
#   $ ./list_lab.py
#
# (At least on OS-X and Linux).
# – you do that with this command:
#
#   $ chmod +x list_lab.py
#
# (The +x means make this executable).
# The file will also need this on the first line:
#
#   #!/usr/bin/env python3
#
# This is known as the "she-bang" line – it tells the shell how to execute that
# file – in this case, with python3
#
# NOTE: on Windows, there is a python launcher which, if everything is
# configured correctly will look at that line to know you want python3 if there
# is more than one python on your system.
#
# If this doesn’t work on Windows, just run the file some other way:
#
#   $ python list_lab.py; OR
#   with run in ipython; OR
#   from your IDE or editor is you are using one
#
# Add the file to your clone of the repository and commit changes frequently
# while working on the following tasks. When you are done, push your changes to
# GitHub and issue a pull request.
#
# (if you are s"ll struggling with git – just write the code for now).
#
# When the script is run, it should accomplish the following four series of
# actions:
#
# Series 1
# --------
#   [1] Create a list that contains "Apples", "Pears", "Oranges" and "Peaches".
#   [2] Display the list (plain old print() is fine...).
#   [3] Ask the user for another fruit and add it to the end of the list.
#   [4] Display the list.
#   [5] Ask the user for a number and display the number back to the user and
#       the fruit corresponding to that number (on a 1-is-first basis).
#       Remember that Python uses zero-based indexing, so you will need to
#       correct.
#   [6] Add another fruit to the beginning of the list using "+" and display
#       the list.
#   [7] Add another fruit to the beginning of the list using insert() and
#       display the list.
#   [8] Display all the fruits that begin with "P", using a for loop.
#
# Series 2
# --------
#   Using the list created in series 1 above:
#   [1] Display the list.
#   [2] Remove the last fruit from the list.
#   [3] Display the list.
#   [4] Ask the user for a fruit to delete, find it and delete it.
#   [5] (Bonus: Multiply the list times two. Keep asking until a match is
#       found. Once found, delete all occurrences.)
#
# Series 3
# --------
#   Again, using the list from series 1:
#   [1] Ask the user for input displaying a line like "Do you like apples?" for
#       each fruit in the list (making the fruit all lowercase).
#   [2] For each "no", delete that fruit from the list.
#   [3] For any answer that is not "yes" or "no", prompt the user to answer
#       with one of those two values (a while loop is good here) Display the
#       list.
#
# Series 4
# --------
#   Once more, using the list from series 1:
#   [1] Make a new list with the contents of the original, but with all the
#       letters in each item reversed.
#   [2] Delete the last item of the original list. Display the original list
#       and the copy.
#
# # Implementation:
# ---------------
#
#   - Solutions presented in 4 steps, one for each series. Runs interactively
#     from command line.
#
#   - Function names, variables, block comments, line continuation, and
#     operator spacing per PEP8 guidelines. Checked with flake8.
#
# Script Usage:
# -------------
#
# 	./list_lab.py
#
# Issues:
# -------
#
#   -
#
# History:
# --------
# 000/2020-10-01/sal/Created.
# =============================================================================

# [1] Series 1
# ----------------------------------------------------------------------------
# Assumption: List development is sequential.
# Assumption: A degree of freedom is allowed for developing command-line UI.
# Assumption: Operations on common list may be destructive.

def validated_menu_choice(choice_in, option_count):
    """
    validated_menu_choice(<choice_in>, <option_count>)
    --------------------------------------------------
    For problem [1-5]. Validate the user menu choice, returning an
    actionable int value.

    Entry: <choice_in>    ::= (str) Menu option number.
           <option_count> ::= (int) Number of menu options.
    Exit:  Returns...
           * int equivalent for 1 <= <choice_in> <= <option_count>; OR
           * 0 if <choice_in> = 'X'; OR
           * -1 for anything else,
    """
    choice_out = (0 if (choice_in.upper() == "X") else -1)
    if (choice_in.isnumeric()):
        choice = int(float(choice_in))
        if ((choice > 0) and (choice <= option_count)):
            choice_out = choice
        else:
            choice_out = -1
    return choice_out

def make_option_menu(options):
    """
    make_option_menu(<options>)
    ---------------------------
    For problem [1-5]. Make an enumerated str of menu options.

    Entry: <options> ::= (list(str*)) menu options, count > 0.
    Exit:  Returns a displayable str of the menu options prefixed with
           a menu option number starting with 1.
    """
    option_menu = ""
    option_count = len(options)
    if (option_count > 0):
        option_menu = "[1] {}".format(options[0])
        for n in range(1, option_count):
            option_menu = \
                option_menu + ", [{}]: {}".format((n + 1), options[n])
    return option_menu

def choose_option_from_menu(option_menu, option_count):
    """
    choose_option_from_menu(<option_menu>, <option_count>)
    ------------------------------------------------------
    For problem [1-5]. Present and respond to user menu choice.

    Entry: <option_menu>  ::= (str) Displayable menu options prefixed
                              with a menu option number starting with 1.
           <option_count> ::= (int) Number of menu options.
    Exit:  Returns int representing user choice.
    """
    prompt = "\n{}\nSelect by item number or X to eXit: "
    choice = input(prompt.format(option_menu))
    return validated_menu_choice(choice, option_count)

def select_from_menu(options):
    """
    select_from_menu(<options>)
    ---------------------------
    For problem [1-5]. Prompt user for a menu choice.

    Entry: <options> ::= (list(str*)) menu options, count > 0.
    Exit:  4 possible outcomes based on user choice...

           (1) Print 'Nothing on the menu today' and exit; OR
           (2) Print 'Exiting from menu' and exit; OR
           (3) Print 'Invalid choice <choice>' and ask again; OR
           (4) Print 'Selection is <choice> (<selection>)' and ask
               again.

           where <choice> is the menu option selection and
           <selection> is the name of the selection.
    """
    selection = ""
    option_count = len(options)
    if (option_count > 0):
        option_menu = make_option_menu(options)
        choice = -1
        while (choice < 0):
            choice = choose_option_from_menu(option_menu, option_count)
            if (choice == 0):
                print("\nExiting from menu")
                break
            elif (choice > 0):
                selection = options[choice - 1]
                print("\nSelection is #{} (\"{}\")".format(choice, selection))
                choice = -1
            else:
                print("\nInvalid choice \"{}\"".format(choice))
    else:
        print("\nNothing on the menu today")

def string_equal(a, b, ignore_case):
    """
    string_equal(<a>, <b>, <ignore_case>)
    -------------------------------------
    For problem [1-8]. Are two strings equivalent?

    Entry: <a>, <b>      ::= (str) Values to compare.
           <ignore_case> ::= (Boolean) When True, ignore char case.
    Exit:  Returns True if <a> and <b> are equivalent.
    """
    result = (a == b)
    if (ignore_case):
        result = (a.lower() == b.lower())
    return result

def collectp_via_loop(element_list):
    """
    collectp_via_loop(<element_list>)
    ---------------------------------
    For problem [1-8]. Iterate on a list of str, collecting values
    beginning with the letter 'p'.

    Entry: <element_list> ::= (list(str*))
    Exit:  Returns a list containing elements of <element_list>
           starting with the letter 'p'.
    """
    pfiltered = []
    for e in element_list:
        if (string_equal(e[0:1], "p", True)):
            pfiltered.append(e)
    return pfiltered

def collectp_via_filter(element_list):
    """
    collectp_via_filter(<element_list>)
    -----------------------------------
    For problem [1-8]. Collect values of a list of str with the filter
    function checking for the first letter as 'p'.

    Entry: <element_list> ::= (list(str*))
    Exit:  Returns a list containing elements of <element_list>
           starting with the letter 'p'.
    """
    return list(filter(lambda e: string_equal(e[0:1],
                       "p", True),
                       element_list))

# ----------------------------------------------------------------------------
# _Fruits - common data between series 1 thru 4
# ----------------------------------------------------------------------------

_Fruits = []

def reset_fruits():
    """
    Reset the global container _Fruits.
    """
    global _Fruits
    _Fruits = ["Apples", "Pears", "Oranges", "Peaches"]

def member_of(list_to_search, element):
    """
    Is element and member of the list?
    """
    found = False
    try:
        found = (list_to_search.index(element) >= 0)
    except Exception:
        found = False
    return found

# [1] Series 1
# ----------------------------------------------------------------------------
# Assumption: List development is sequential.
# Assumption: A degree of freedom is allowed for developing command-line UI.
# Assumption: Operations on common list may be destructive.

def series_1():
    """
    series_1()
    ----------
    Satisfy Series-1 requirements 1 through 8.

    Entry: -
    Exit:  Multiple input and output events. Modifies content of global
           var _Fruits.
    """
    global _Fruits
    print("\nSeries-1 Solutions\n")
    print("{}".format("-" * 50))
    #
    # [1-1] ...creating the initial list.
    #
    reset_fruits()
    #
    # [1-2] ...printing the results of [1-1].
    #
    print("\nfruits are {}".format(_Fruits))
    #
    # [1-3] ...also added a new element check and some formatting on the
    #       input.
    #
    is_new_fruit = False
    while(not is_new_fruit):
        next_fruit = input("\n=>Enter a new fruit: ")
        is_new_fruit = (not member_of(_Fruits, next_fruit))
    next_fruit = next_fruit.replace("\"", "").strip().capitalize()
    _Fruits.append(next_fruit)
    #
    # [1-4] ...printing the results of [1-3].
    #
    print("\nfruits are {}".format(_Fruits))
    #
    # [1-5] ...a slightly more sophisticated fruit selector.
    #
    select_from_menu(_Fruits)
    #
    # [1-6] ...adding to the head of a list and printing the results.
    #
    print("\nInsert an item at the head of a list using '+' operator:")
    f = "Kiwi"
    more_fruits = [f] + _Fruits
    print(f"\n[{f}] + {_Fruits} = {more_fruits}")
    _Fruits = more_fruits
    #
    # [1-7] ...adding to the head of a list and printing the results (again).
    #
    print("\nInsert an item at the head of a list using the 'insert' method:")
    f = "Pomegranate"
    initial_fruits = _Fruits.copy()
    _Fruits.insert(0, f)
    print(f"\n{initial_fruits}.insert(0,\"{f}\") = {_Fruits}")
    #
    # [1-8] ...filtering via loop (and by the filter function).
    #
    print("\nFiltering fruits starting with 'P' (upper or lower case):")
    pfruits = collectp_via_loop(_Fruits)
    print(f"\ncollectp_via_loop({_Fruits}) = {pfruits}")
    pfruits = collectp_via_filter(_Fruits)
    print(f"\ncollectp_via_filter({_Fruits}) = {pfruits}")
    print("\nSeries-1 Done")

# [2] Series 2
# ----------------------------------------------------------------------------
# Assumption: List development is sequential.
# Assumption: A degree of freedom is allowed for developing command-line UI.
# Assumption: Operations on common list may be destructive.

def delete_fruit_by_name(fruits_in):
    """
    """
    fruit_to_delete = ""
    fruit_exists = False
    print(f"\nfruits_in = {fruits_in}")
    if (len(fruits_in) > 0):
        while(not fruit_exists):
            fruit_to_delete = input("\nWhich fruit shall be deleted? ")
            fruit_to_delete\
                = fruit_to_delete.replace("\"", "").strip().capitalize()
            fruit_exists = any(string_equal(fruit, fruit_to_delete, True)
                               for fruit in fruits_in)
            if (not fruit_exists):
                print("\nThat fruit does not exist. Pick one from the list.")
        while fruit_exists:
            fruits_in.remove(fruit_to_delete)
            fruit_exists = any(string_equal(fruit, fruit_to_delete, True)
                               for fruit in fruits_in)
        # [3] Display the list.
        print("\ndelete_fruit_by_name(fruits_in) = {}".format(fruits_in))
    else:
        print("\nThere is no fruit in the fruits list.")
    return fruits_in

def series_2():
    """
    series_2()
    ----------
    Satisfy Series-2 requirements 1 through 5.
    """
    global _Fruits
    reset_fruits()
    print("\nSeries-2 Solutions")
    print("{}".format("-"*50))
    #
    # [1] Display the last state of list fruits.
    #
    print(f"\nfruits = {_Fruits}")
    #
    # [2] Remove the last fruit from the list.
    #
    last_index = len(_Fruits) - 1
    del(_Fruits[last_index])
    #
    # [3] Display the list.
    #
    print("\ndel(fruits[{}])\n\nfruits = {}".format(last_index, _Fruits))
    #
    # [4] Remove a fruit from list by name.
    #
    existing_fruits = _Fruits.copy()
    delete_fruit_by_name(existing_fruits)
    #
    # [5] Add duplicates of every item in list and repeat task [4] so that
    #     all occurrences of fruit are removed.
    #
    double_fruits = existing_fruits.copy()*2
    delete_fruit_by_name(double_fruits)
    #
    # Set shared list to last working list.
    #
    _Fruits = double_fruits
    print("\nSeries-2 Done")

# [3] Series 3
# ----------------------------------------------------------------------------
# Assumption: List development is sequential.
# Assumption: A degree of freedom is allowed for developing command-line UI.
# Assumption: Operations on common list may be destructive.

# --------------------------
# Series 3 command constants
# --------------------------
_SER3_CMD_YES = ":yes"
_SER3_CMD_NO = ":no"
_SER3_CMD_EXIT = ":exit"

def response_to_command(user_response):
    """
    response_to_command(<user_response>)
    ------------------------------------
    Convert a user response to a function command.

    Entry: <user_response> ::= (str) from input function.
    Exit:  Returns _SER3_CMD_YES for <user_response> like "y*"; OR
           _SER3_CMD_NO for <user_response> like "n*"; OR
           _SER3_CMD_EXIT for <user_response> like "ex*"; OR
           "" for all other input.
    """
    global _SER3_CMD_YES
    global _SER3_CMD_NO
    global _SER3_CMD_EXIT
    result = "?"
    if (string_equal(user_response[0:1], "y", True)):
        result = _SER3_CMD_YES
    elif (string_equal(user_response[0:1], "n", True)):
        result = _SER3_CMD_NO
    elif (string_equal(user_response[0:2], "ex", True)):
        result = _SER3_CMD_EXIT
    else:
        result = ""
    return result

def equalp(a, b, ignore_case):
    """
    equalp(<a>, <b>, <ignore_case>)
    -------------------------------------
    For problem [1-8]. Are two values equivalent?

    Entry: <a>, <b>      ::= (str) Values to compare.
           <ignore_case> ::= (Boolean) When True, ignore char case when
                             both <a> and <b> are strings.
    Exit:  Returns True if <a> and <b> are equivalent.
    """
    if ((type(a) is str) and (type(b) is str)):
        result = string_equal(a, b, ignore_case)
    else:
        result = (a == b)
    return result

def delete_from_list(list_in, element_to_delete):
    """
    delete_from_list(<list_in>, <element_to_delete>)
    ------------------------------------------------
    Delete all occurrences of a value from a list.

    Entry: <list_in>           ::= (list) to delete from.
           <element_to_delete> ::= Value to delete.
    Exit:  Returns <list_in> with <element_to_delete>) removed.
    """
    element_exists = any(equalp(element, element_to_delete, True)
                         for element in list_in)
    while element_exists:
        list_in.remove(element_to_delete)
        element_exists = any(equalp(element, element_to_delete, True)
                             for element in list_in)
    return list_in

def get_user_command(subject):
    """
    get_user_command(<subject>)
    ---------------------------
    Get command response for dialog response to 'Do you like <subject>
    ([Y]es, [N]o, or e[X]it)?'

    Entry: <subject> ::= (str) subject of prompt
    Exit:  Returns _SER3_CMD_YES, _SER3_CMD_NO, _SER3_CMD_EXIT, or blank.
    """
    user_response = input("\nDo you like {} ([Y]es, [N]o, or e[X]it)? "
                          .format(subject.lower()))
    user_response = user_response.replace("\"", "").strip().lower()
    user_cmd = response_to_command(user_response)
    return user_cmd

def series_3():
    """
    series_3()
    ----------
    Satisfy Series-3 requirements 1 through 3.

    Entry: -
    Exit:  Multiple input and output events.
    """
    global _Fruits
    reset_fruits()
    valid_commands = (_SER3_CMD_YES, _SER3_CMD_NO, _SER3_CMD_EXIT)
    print("\nSeries-3 Solutions")
    print("{}".format("-"*50))
    #
    # [1] Query for user fruit fondness
    #
    local_fruits = _Fruits.copy()
    print("\nfruits = {}".format(local_fruits))
    unique_fruits = set(local_fruits)
    for unique_fruit in unique_fruits:
        user_cmd = get_user_command(unique_fruit)
        while (not (user_cmd in valid_commands)):
            #
            # [3] Limit user responses to "yes", "no"", and "exit" (last valid
            #     response allows user to bail out of processing every unique
            #     fruit, which can be tedious.)
            #
            print("\nInvalid response.")
            user_cmd = get_user_command(unique_fruit)
        if (user_cmd == _SER3_CMD_YES):
            #
            # No action for "yes" or "y*".
            #
            continue
        elif (user_cmd == _SER3_CMD_NO):
            #
            # [2] When the user response is "no". Also works for "na" and
            #     "nyet" and "non".
            #
            print(f"\nDeleting {unique_fruit}")
            local_fruits = delete_from_list(local_fruits, unique_fruit)
            print(f"\nfruits = {local_fruits}")
        elif (user_cmd == _SER3_CMD_EXIT):
            #
            # User entered "ex*". Exit loop.
            #
            break
        else:
            #
            # This should never be reached but I always include an 'else' in
            # an if-elif-else block.
            #
            continue
    print(f"\nRemaining fruits = {local_fruits}")
    print("\nNo fruit was bruised in the running of this function.")
    print("\nSeries-3 Done")

# [4] Series 4
# ----------------------------------------------------------------------------
# Assumption: List development is sequential.
# Assumption: A degree of freedom is allowed for developing command-line UI.
# Assumption: Operations on common list may be destructive.

def reverse_string(s):
    return s[::-1]

#   [1] Make a new list with the contents of the original, but with all the
#       letters in each item reversed.
#   [2] Delete the last item of the original list. Display the original list
#       and the copy.

def series_4():
    """
    series_4()
    ----------
    Satisfy Series-4 requirements 1 through 2.

    Entry: -
    Exit:  Multiple input and output events.
    """
    global _Fruits
    reset_fruits()
    print("\nSeries-4 Solutions")
    print("{}".format("-"*50))
    original_list = _Fruits.copy()
    #
    # [1] Original list with chars in string elements reversed.
    #
    print(f"\noriginal_list = fruits.copy() = {original_list}")
    rev_element_list = list(map(reverse_string, original_list))
    print("\nrev_element_list = list(map(reverse_string,original_list)) = {}"
          .format(rev_element_list))
    #
    # [2] Original list with last element removed.
    #
    copied_list = original_list.copy()
    print("\ncopied_list = {}".format(copied_list))
    del copied_list[-1]
    print("\ndel copied_list[-1]")
    print("\ncopied_list = {}".format(copied_list))
    print("\noriginal_list = {}".format(original_list))

if (__name__ == "__main__"):
    #
    # Satisfies overall requirement for command-line execution.
    #
    series_1()
    series_2()
    series_3()
    series_4()


# -*- coding: utf-8 -*-
"""
Created on Tue Oct  4 12:46:30 2016
Dave plays around with Bosch data
@author: Dave
"""

import pandas as pd
import os

dates = pd.read_csv('train_date.csv',
                    nrows = 10000,
                    index_col = 0,
                    dtype = pd.np.float64,)

numeric = pd.read_csv('train_numeric.csv',
                    nrows = 10000,
                    index_col = 0,
                    dtype = pd.np.float32,)


import os
import subprocess
import ctypes
import sys

# 如何获得管理员权限
# https://blog.csdn.net/qq_17550379/article/details/79006655
# https://blog.csdn.net/qq_17550379/article/details/79006718
# print(ctypes.windll.shell32.IsUserAnAdmin())

# 1. 使用 os.system 可以直接调用系统命令
# 返回值表示程序执行是否成功
os.system('echo hello world')
ret = os.system('aaaa')
print(ret)

# 2. os.popen 可以获取命令的输出
py_ver = os.popen('python -V').read()
print(py_ver)

# 3. subprocess 用来运行其他程序的库
# 参考资料：https://www.cnblogs.com/zhoug2020/p/5079407.html
# 3.1 call 类似于 os.system 更多参数和调用方式
subprocess.call(['dir', '.'], shell=True, cwd='C:/proj')
subprocess.call('echo -----------------------------', shell=True)

# 3.2 Popen 可以用于交互
#     os.popen 也使用了 subpress
#     实时输出 https://blog.csdn.net/u012206617/article/details/84560895
cmd = subprocess.Popen('cmd.exe', shell=True,
                       stdin=subprocess.PIPE, stdout=subprocess.PIPE)
cmd.stdin.write(b'ping www.baidu.com\r\n')
cmd.stdin.write(b'exit\r\n')  # 如果不退出后边会卡在 stdout.read
cmd.stdin.flush()
# 使用 poll 检查子进程有没有关闭
while cmd.poll() is None:
    line = cmd.stdout.readline()
    # Popen 有 encodeing 参数还没试
    print(line.decode('gbk'), end='')

#  3.2 check_call 执行并检查返回值，失败抛出异常
try:
    subprocess.check_call('abcdef',shell=True)
except subprocess.CalledProcessError as e:
    print(e)
    print(e.returncode)


'''
公開情報からパーシャルチェックリストを作成する

update_ctestwin_pck.py
2021.Oct.25 7M4MON

2020年現在、日本のアマチュア無線局数はおよそ40万局ですが、
このうち、JARLの国内4大コンテストに 2015-2019の5年間で書類を提出した局は、
およそ6000局（7M4MON 調べ）です。
コールサインのアルファベット・数字の組み合わせは膨大な数がありますが、
コンテストに参加する局は、ほとんど上記の6000局のコールサインです。
そのため、過去の交信履歴等からある程度の予測と省力化が可能です。
コンテストロギングソフトでは、コールサインの一部を入力すると
交信相手のコールサイン候補が表示されるパーシャルチェックという機能があります。
パーシャルチェックではコールサインだけではなくコンテストナンバーの候補も表示されます。
上記の国内4大コンテストのコンテストナンバーは RS(T) ＋ 運用地の都府県・北海道地域番号または
市郡区番号＋空中線電力を表すアルファベット１文字です。
総務省の無線免許状検索ではコールサインで検索すると常置場所の市区町村名まで分かりますので、
コンテスト書類提出局のコールサインとつき合わせて、パーシャルチェックのデータベースを作成しました。

* プログラムの流れ
1.コンテストの結果ページを読み込んで、コールサインを抜き出す
2.既存の ctestwin.pck を読み込んで、コールサインを抜き出す
3. 1と2を結合し、重複を削除し、ソートする
4.総務省の免許状検索データベースから常置場所を取得する
5.常置場所の市町村名から市郡区番号を取得する
6. 3と4と5を結合して、ctestwin.pckを作成

以前は新規参加局を追記していくスタイルでしたが、時間経過とともに
廃局されたり常置場所を変更されたりする局が増えてきますので、
全てのコールサインを再検索して、最新の常置場所に置き換えるとともに
免許状検索に登録されていない局はリストから削除する方針に変更しました。
'''

contest_result_url = "https://www.jarl.org/Japanese/1_Tanoshimo/1-1_Contest/6m/2021/entry.html"
read_pck_filename = "./ctestwin_20211024.pck"       # 前回作成したパーシャルチェックファイル名
write_pck_filename = "./ctestwin_20211122.pck"      # 今回作成するパーシャルチェックファイル名
new_callsign_filename = "./2021_6m_new.txt"      # 今回のコンテストの新規参加局
read_acag_filename = './pref_acag.csv.txt'          # 市町村郡名とコードの対照表（.csvだとExcelで開いたときに先頭の 0 が消えるのを嫌って .txtにしてある)
wait_sec = 3                                        # 過負荷をかけないようにするための、1件あたりのウェイト時間。3のとき4878件で5時間半程度。
pass_idx = 0                                        # 中断したときに飛ばすインデックス番号

import requests, re, json, time, csv, datetime

'''コールサイン抜き出し用正規表現'''
cs_re = '[J|7][A-S][0-9][A-Z]+? '           # Jまたは7で始まり、2文字目がAからSのアルファベット、3文字目が数字、4文字目からアルファベットが続き、末尾にデリミタの半角スペース
#cs_re = ' (J[A-S][0-9]|[7-8])[A-Z]+? '     #サクラエディタではOKだがpythonだとだめ
#cs_re = ' (J[A-S][0-9][A-Z]+?|[7-8][A-Z]+?) '

'''コンテストの結果ページからコールサインを抜き出す（末尾に半角スペースあり）'''
def read_callsign_resultpage(url):
    res = requests.get(url)
    callsign_list = re.findall(cs_re,res.text)
    return callsign_list

'''過去の ctestwin.pck からコールサインを抜き出す（末尾に半角スペースあり）'''
def read_callsign_pckfile(pck_filename):
    pckfile = open(pck_filename, 'r')
    pckstr = pckfile.read()
    pckfile.close()
    callsign_list_pck = re.findall(cs_re,pckstr)
    return callsign_list_pck

'''コンテストの参加局で過去のデータベースに登録のない局を抜き出す'''
def pickup_new_cs(test_cs, pck_cs):
    new_cs = []
    for cs in test_cs:
        if (cs in pck_cs) == False :
            new_cs.append(cs)
    return new_cs

'''リストをダンプして保存'''
def dump_list(the_list, filename):
    f = open(filename, 'w')
    for x in the_list:
        f.write(str(x) + "\n")
    f.close()

''' コールサインを総務省の無線局等情報検索Web-APIで検索 '''
''' 応答結果から無線局の設置場所取得して市町村名を返す '''
''' get_qth.py から移植'''
# res_txt = {"musen":[{"listInfo":{"name":"＊＊＊＊＊（7M4MON）","radioStationPurpose":"アマチュア業務用","tdfkCd":"東京都中央区","no":"1","licenseDate":"2018-04-02"}}],"musenInformation":{"totalCount":"1","lastUpdateDate":"2020-08-02"}}
def get_city(callsign):
    response = requests.get('https://www.tele.soumu.go.jp/musen/list?ST=1&OF=2&OW=AT&DA=0&DC=1&SC=1&MA=' + callsign)
    city = ""
    if response.status_code == 200:
        res_txt =response.text    # レスポンスのHTMLを文字列で取得
        if 'tdfkCd' in res_txt:   # 存在（廃局）チェック
            d = json.loads(res_txt)                     # Json形式の文字列をdictにパース
            city = d['musen'][0]['listInfo']['tdfkCd']  # tdfkCd キーの中身を取得
            print(city)
        else :
            pass    #コールサインがデータベースにない
    else:
        pass        #サーバー応答なし
    return city

'''市町村名から市郡コードを返す'''
'''get_cgnum.py から移植'''
cgnum_list = [] # 町村郡名とコードの対照表を保持しておくリスト
# 北海道のダブリ郡の例外処理。コードが後の郡の町名だったらコードを直す。
hokkaido_dupe_code = ['01006','01014','01022','01042','01045','01050','01073']  # 先にあるコードのリスト
abuta_iburi = ['豊浦町','洞爺湖町']
uryu_kamikawa = ['幌加内町']    #例外中の例外
kamikawa_kamikawa = ['鷹栖町','東神楽町','当麻町','比布町','愛別町','上川町','東川町','美瑛町','和寒町','剣淵町','下川町']
sorachi_kamikawa = ['上富良野町','中富良野町','南富良野町']
teshio_souya = ['豊富町','幌延町']
nakagawa_tokachi = ['幕別町','池田町','豊頃町','本別町']
yuufutsu_kamikawa = ['占冠村']
hokkaido_dupe_towns = [abuta_iburi, uryu_kamikawa,kamikawa_kamikawa,sorachi_kamikawa,teshio_souya,nakagawa_tokachi,yuufutsu_kamikawa]
hokkaido_dupe_county = ['北海道虻田郡(胆振)','北海道雨竜郡(上川)','北海道上川郡(上川)','北海道空知郡(上川)','北海道天塩郡(宗谷)','北海道中川郡(十勝)','北海道勇払郡(上川)']

def get_cgnum(city_name):
    # 町村名は不要なので〇〇郡までを抜き出す。
    if '郡' in city_name:   # 愛知県蒲郡市, 福岡県小郡市, 奈良県大和郡山市, 福島県郡山市も含まれてしまうが削っても被らないので問題なし
        full_city_name = city_name  #バックアップ
        city_name = full_city_name[0:city_name.find('郡') + 1]  # 前方から検索
        town_name = full_city_name[city_name.find('郡') + 1:]
        print (town_name)

    cgnum_str = ""
    list_hit = False

    # 郡市区名を探す
    for cg_item in cgnum_list:
        if city_name in cg_item[1]:
            list_hit = True
            break

    if list_hit == True:    # （表記ゆれや合併などで名前が変わってなければ）必ずあるはず
        cgnum_str = cg_item[0]
        country_name = cg_item[1]
        '''北海道のダブリ郡を処理する'''
        if cgnum_str in hokkaido_dupe_code :
            print('dupe code:' + country_name)
            dupe_index = hokkaido_dupe_code.index(cgnum_str)
            if town_name in hokkaido_dupe_towns[dupe_index] :   #例外の町がある
                country_name = hokkaido_dupe_county[dupe_index]
                print('北海道の例外')
                if dupe_index == 1 : # 例外中の例外 北海道雨竜郡(上川)幌加内町
                    cgnum_str = '01081'
                    print ('中の例外 北海道雨竜郡(上川)幌加内町')
                else:
                    cgnum_str = str(int(cgnum_str) + 1).zfill(5) # 幌加内町以外は＋１すれば良い
    else:   #not in list
        pass

    print(cgnum_str + '\n')
    return cgnum_str

'''処理開始時に市郡コードを cgnum_list[] にロードしておく'''
def load_cgnum_list():
    with open(read_acag_filename,'r',encoding="SHIFT-JIS") as f_in:
        reader = csv.reader(f_in)
        for row in reader:
            cgnum_list.append(row)

'''メインの処理'''
if __name__ == '__main__':
    start_date = datetime.datetime.now()
    cs_list = read_callsign_resultpage(contest_result_url)
    cs_list.sort()
    print("対象局数: " + str(len(cs_list)))
    cs_list_pck = read_callsign_pckfile(read_pck_filename)
    print("既登録局数: " + str(len(cs_list_pck)))
    cs_new = pickup_new_cs(cs_list, cs_list_pck)
    print("新規局数: " + str(len(cs_new)))
    dump_list(cs_new, new_callsign_filename)
    cs_list.extend(cs_list_pck)
    dup = [x for x in set(cs_list) if cs_list.count(x) > 1]
    print("重複局数: " + str(len(dup)))
    uniq_cs_list = list(set(cs_list))       #重複を削除
    uniq_cs_list.sort()
    print("実行件数: " + str(len(uniq_cs_list)))
    fw = open(write_pck_filename, 'a')
    load_cgnum_list()
    i = 0
    qrt = 0
    no_cgnum = 0
    for cs in uniq_cs_list:
        if pass_idx > i:
            i += 1
            pass
        else :
            callsign = cs.strip()   #改行やスペースの削除
            print(str(i) + " " + callsign)
            city_name = get_city(callsign)
            if city_name != "" :
                cgnum_str = get_cgnum(city_name)
                print(callsign + ' ' + cgnum_str + ' '+ city_name , file=fw)
                if cgnum_str == "" :
                    no_cgnum += 1   #表記ゆれや合併などで市町村名が変わっている
            else:
                qrt += 1            #廃局などでコールサインが総務省のデータベースにない
            time.sleep(wait_sec)    #総務省のデータベースに過負荷をかけないようにちょっと待つ
            i += 1
    print('IDX:' + str(i) + ' QRT:' + str(qrt) + ' NIL:'+ str(no_cgnum) , file=fw)    #最後に結果を追記
    fw.close()
    stop_date = datetime.datetime.now()
    print('処理時間: '+ str(stop_date - start_date ))