import datetime
import logging
from typing import List, Tuple, Iterable

import googleapiclient
from django.utils import timezone

from amelie.claudia.clau import Claudia
from amelie.claudia.google import GoogleSuiteAPI
from amelie.claudia.models import Mapping, Event, DrivePermission, AliasGroup
from amelie.claudia.plugins.plugin import ClaudiaPlugin
from django.conf import settings

logger = logging.getLogger(__name__)

DELETE_GSUITE_USER = 'DELETE_GSUITE_USER'

class GoogleSuitePlugin(ClaudiaPlugin):
    def __init__(self):
        self.google = GoogleSuiteAPI()

    # ======= Deletion scheduling functions =======

    def schedule_delete(self, mp):
        """
        Schedule the account for deletion, if it was not marked already.
        :type mp: Mapping
        """
        if self.delete_scheduled(mp):
            logger.debug("User already scheduled for delete")
        else:
            expiration = timezone.now() + datetime.timedelta(30)
            Event(type=DELETE_GSUITE_USER, mapping=mp, execute=expiration).save()
            return True

    def unschedule_delete(self, mp):
        """
        Unschedule the account for deletion
        :type mp: Mapping
        """
        if self.delete_scheduled(mp):
            Event.objects.filter(type=DELETE_GSUITE_USER, mapping=mp).delete()
            return True

    def delete_scheduled(self, mp):
        """
        Check if user is scheduled for deletion.
        :type mp: Mapping
        """
        return Event.objects.filter(type=DELETE_GSUITE_USER, mapping=mp).exists()

    # ======= Claudia functions =======

    def verify_mapping(self, claudia: Claudia, mp: Mapping, fix: bool = False) -> None:
        changes = []

        logger.debug("Verifying google suite object {object_name} (cid: {cid})"
                     .format(object_name=mp.name, cid=mp.id))

        if not fix:
            logger.debug("No changes will be made to internal or external accounts.")

        # If mapping has gsuite id, get its account
        account_group_memberships = []
        if mp.get_gsuite_id() is not None:
            if mp.is_person():
                account = self.google.get_user(mp)
                account_group_memberships = self.google.get_group_memberships(mp)
            elif mp.is_group():
                account = self.google.get_group(mp)
            elif mp.is_contact():
                # Contacts only have a GSuite ID to be able to delete them from groups, they don't really have accounts.
                # This entry is only here to avoid hitting the nasty ValueError below.
                account = None
            elif mp.is_shareddrive():
                account = self.google.get_shared_drive(mp)
            else:
                raise ValueError("Mapping {} (cid {}) is not a group, but not a person either?".format(mp.name, mp.id))
        else:
            account = None

        # If mappging is person, a delete is scheduled, but they need an account and have a username and e-mail,
        # remove the scheduled deletion
        if mp.is_person() and self.delete_scheduled(mp) and mp.needs_gsuite_account():
            logger.debug("User is active again, removing scheduled delete")
            if fix and self.unschedule_delete(mp):
                claudia.notify_gsuite_unscheduled_delete(mp)

        # If we could not find an account, but the mapping had a gsuite id and it is not a contact, remove the gsuite id
        if account is None and mp.get_gsuite_id() and not account_group_memberships and not mp.is_contact():
            logger.debug("Removing invalid GSuite ID for {}".format(mp.name))
            if fix:
                mp.set_gsuite_id(None)

        # If the user needs an account and has a username and e-mail, but no account, create one.
        if account is None and mp.needs_gsuite_account():
            if mp.is_person():
                if mp.adname:

                    # Need to create acount but first check if person is in any groups by its emailaddress
                    # If it is, remove it from the groups and create the account. The group memberships will be
                    # fixed later by the verification process.
                    logger.debug("Want to create account for {}, but first need to check old group memberships.".format(mp.name))
                    current_group_memberships = self.google.get_group_memberships(mp)
                    if not current_group_memberships:
                        logger.debug("{} has no group memberships on their e-mail address".format(mp.name))
                    for membership in current_group_memberships:
                        logger.debug("Removing old membership of e-mail {} from group {} <{}>".format(
                            mp.email, membership['name'], membership['email']
                        ))
                        self.google.remove_group_member(mp, membership)

                    logger.debug("Object {} had no GSuite account, creating one with name {}.".format(mp.name, mp.adname))
                    if fix:
                        account = self.google.create_user(mp)
                        if account:
                            changes.append(('gsuite', 'User {} created'.format(account['primaryEmail'])))
                            claudia.notify_gsuite_created(mp)
                else:
                    logger.debug("Not creating GSuite account for {}, because the mapping has no adname.".format(mp))
            elif mp.is_group():
                if mp.email:
                    logger.debug("Object {} had no GSuite group, creating one with email {}.".format(mp.name, mp.email))
                    if fix:
                        account = self.google.create_group(mp)
                        if account:
                            changes.append(('gsuite', 'Group {} <{}> created'.format(account['name'], account['email'])))
                            claudia.notify_gsuite_created(mp)
                else:
                    logger.debug("Not creating GSuite group for {}, because the mapping has no e-mail address.".format(
                        mp
                    ))
            elif mp.is_contact():
                # Contacts don't need an account, they are just an e-mail address in GSuite
                # But we pass here so we don't run into the nasty ValueError below.
                pass
            elif mp.is_shareddrive():
                logger.debug("Object {} is a new shared drive, creating shared drive.".format(mp.name))
                if fix:
                    account = self.google.create_shared_drive(mp)
                    if account:
                        changes.append(('gsuite', 'Shared Drive {} created'.format(account['name'])))
                        claudia.notify_gsuite_created(mp)
            else:
                raise ValueError("Mapping {} (cid {}) is not a group, but not a person or contact either?".format(
                    mp.name, mp.id))

        # If the user has an account, check if we need one. If not, schedule deletion if person, and delete if group
        if account is not None:
            delete_reason = None
            if not mp.needs_gsuite_account():
                if mp.is_person() and not mp.adname:
                    delete_reason = "no longer has an account name"
                elif not mp.email:
                    delete_reason = "no longer has an e-mail address"
                else:
                    delete_reason = "is no longer active"

            if delete_reason is not None:
                if mp.is_group():
                    # Delete groups right away
                    logger.debug("Group {} {}, deleting GSuite group.".format(mp.name, delete_reason))
                    changes.append(('gsuite', 'Group {} deleted'.format(mp.name, mp.email)))
                    if fix:
                        self.google.delete_group(mp)
                        account = None
                        mp.set_gsuite_id(None)
                elif mp.is_person():
                    logger.debug("User {} {}, scheduling GSuite account deletion.".format(mp.name, delete_reason))
                    # Schedule persons for later deletion
                    if fix and self.schedule_delete(mp):
                        claudia.notify_gsuite_scheduled_delete(mp)
                else:
                    raise ValueError("Mapping {} (cid {}) is not a group, "
                                     "but not a person either?".format(mp.name, mp.id))

        # If we have an account or this is a contact or a person, check the details
        if account:
            # If it is a person, check the details for a User in GSuite
            if account and mp.is_person():
                changes += self.verify_mapping_person(claudia=claudia, mp=mp, account=account, fix=fix)

            # If it is a group, check the details for a Group in GSuite
            elif mp.is_group():
                changes += self.verify_mapping_group(claudia=claudia, mp=mp, account=account, fix=fix)

            # If it is a shared drive, check the name and permissions
            elif mp.is_shareddrive():
                changes += self.verify_mapping_shareddrive(claudia=claudia, mp=mp, account=account, fix=fix)

            # Else, wtf, what is this thing?
            else:
                raise ValueError("Mapping {} (cid {}) is not a group, but not a person either?".format(mp.name, mp.id))

        # Verify group memberships for anything that has an account, is a contact, or is a person (people can
        # be in groups without having an account (e.g. do-group aliases))
        if account or mp.is_contact() or mp.is_person():
            # If it has an account, or is a contact or person with an e-mail address
            if (account or ((mp.is_contact() or mp.is_person()) and mp.email)) and not mp.is_shareddrive():
                # Check if group memberships for the account are correct
                website_aliases = [g for g in mp.groups('mail') if g.is_gsuite_group_active() and g.email]
                website_aliases_emails = [x.email.lower() for x in website_aliases]
                gsuite_groups = self.google.get_group_memberships(mp)
                gsuite_group_emails = [x['email'].lower() for x in gsuite_groups]

                to_add_groups = [x for x in website_aliases if x.email not in gsuite_group_emails]
                to_remove_groups = [x for x in gsuite_groups if x['email'] not in website_aliases_emails]
                matched_groups = [x for x in website_aliases_emails if x in gsuite_group_emails]

                if matched_groups:
                    logger.debug("Matched groups: {}".format(matched_groups))
                if to_add_groups:
                    logger.debug("Should be added to GSuite groups: {}".format([x.email for x in to_add_groups]))
                    changes.append(('gsuite_groups', ["+{}".format(x.email) for x in to_add_groups]))
                if to_remove_groups:
                    logger.debug("Should be removed from GSuite groups: {}".format([x['email'] for x in to_remove_groups]))
                    changes.append(('gsuite_groups', ["-{}".format(x['email']) for x in to_remove_groups]))

                if to_add_groups or to_remove_groups:
                    if fix:
                        for group in to_add_groups:
                            # First check if this group actually exists
                            gs_group = self.google.get_group(group)
                            if not gs_group:
                                # Let Claudia verify the group if it does not exist
                                claudia.do_verify(group)
                                gs_group = self.google.get_group(group)

                            if gs_group:
                                details = self.google.add_group_member(mp, group)
                                # Contacts (and inactive people) are added by e-mail in a group,
                                # but cannot be deleted by e-mail.
                                # They can only be deleted by using the unique ID that they get when adding them, so
                                # we save that as the gsuite ID and use it when deleting it.
                                if details is not None and (mp.is_contact() or (mp.is_person() and not mp.gsuite_id)):
                                    mp.gsuite_id = details['id']
                                    mp.save()
                        for group in to_remove_groups:
                            self.google.remove_group_member(mp, group)

        # Notify if there were changes
        if changes:
            claudia.notify_gsuite_changed(mp, changes)

    def verify_mapping_person(self, claudia: Claudia, mp: Mapping, account, fix: bool = False) -> List[Tuple[str, Iterable[str]]]:
        changes = []

        logger.debug("Checking Person-specific details for {object_name} (cid: {cid})".format(
            object_name=mp.name, cid=mp.id
        ))

        # Check if e-mail address is correct (only if adname is present, or else it would result in an empty e-mail)
        if mp.adname:
            primary_alias = "{}@{}".format(mp.adname, settings.CLAUDIA_GSUITE['PRIMARY_DOMAIN'])
            if account['primaryEmail'] != primary_alias:
                logger.debug("Mail changed to {} (was: {})".format(primary_alias, account['primaryEmail']))
                changes.append(("mail", (account['primaryEmail'], primary_alias)))
                if fix:
                    self.google.update_user_primary_email(mp)

        # Check if First and Last names are correct
        update = False
        if mp.givenname() and mp.surname():  # Needs to have both to be able to check them separately
            if account['name']['givenName'] != mp.givenname():
                logger.debug("Given name changed to {} (was: {})".format(mp.givenname(), account['name']['givenName']))
                update = True
                changes.append(("givenName", (account['name']['givenName'], mp.givenname())))
            if account['name']['familyName'] != mp.surname():
                logger.debug("Surname changed to {} (was: {})".format(mp.surname(), account['name']['familyName']))
                update = True
                changes.append(("familyName", (account['name']['familyName'], mp.surname())))
        else:
            if account['name']['givenName'] != mp.name or account['name']['familyName'] != mp.name:
                logger.debug("Name changed to {} (was: {})".format(mp.name, account['name']['givenName']))
                update = True
                changes.append(("name", (account['name']['givenName'], mp.name)))

        if update and fix:
            self.google.update_user_name(mp)

        # Check if personal aliases are correct (only if the adname is set)
        if mp.adname:
            # Remove adname from aliases because that is our primary email
            aliases = filter(lambda x: x != mp.adname, mp.personal_aliases())
            should_be_aliases = ["{}@{}".format(x, settings.CLAUDIA_GSUITE['PRIMARY_DOMAIN']) for x in aliases]

            # Add extra personal aliases that were manually added to this mapping
            should_be_aliases += [x.email for x in mp.extra_personal_aliases.all()]

            # Add the aliases key into the account if it does not exist
            if 'aliases' not in account:
                account['aliases'] = []

            to_add_aliases = [x for x in should_be_aliases if x not in account['aliases']]
            to_remove_aliases = [x for x in account['aliases'] if x not in should_be_aliases]
            matched_aliases = [x for x in should_be_aliases if x in account['aliases']]

            if matched_aliases:
                logger.debug("Matched personal aliases: {}".format(matched_aliases))
            if to_add_aliases:
                logger.debug("Should be added to personal aliases: {}".format(to_add_aliases))
                changes.append(('personal_aliases', ["+{}".format(x) for x in to_add_aliases]))
            if to_remove_aliases:
                logger.debug("Should be removed from personal aliases: {}".format(to_remove_aliases))
                changes.append(('personal_aliases', ["-{}".format(x) for x in to_remove_aliases]))

            if to_add_aliases or to_remove_aliases:
                if fix:
                    self.google.update_personal_aliases(mp, to_add_aliases, to_remove_aliases)

        return changes

    def verify_mapping_group(self, claudia: Claudia, mp: Mapping, account, fix: bool = False) -> List[Tuple[str, Iterable[str]]]:
        changes = []

        logger.debug("Checking Group-specific details for {object_name} (cid: {cid})".format(
            object_name=mp.name, cid=mp.id
        ))

        # Check group e-mail address
        if account['email'] != mp.email:
            logger.debug("Mail changed to {} (was: {})".format(mp.email, account['email']))
            changes.append(("mail", (account['email'], mp.email)))
            if fix:
                self.google.update_group_email(mp)

        # Check group name
        if account['name'] != mp.name:
            logger.debug("Name changed to {} (was: {})".format(mp.name, account['name']))
            changes.append(("name", (account['name'], mp.name)))
            if fix:
                self.google.update_group_name(mp)

        # Check group description
        if 'description' in mp.extra_data():
            if account['description'] != mp.extra_data()['description']:
                logger.debug("Description changed to {} (was: {})".format(mp.extra_data()['description'],
                                                                          account['description']))
                changes.append(("description", (account['description'], mp.extra_data()['description'])))
                if fix:
                    self.google.update_group_description(mp)

        # Check group aliases
        should_be_aliases = [x.email for x in mp.extra_personal_aliases.all()]

        # Add the aliases key into the account if it does not exist
        if 'aliases' not in account:
            account['aliases'] = []

        to_add_aliases = [x for x in should_be_aliases if x not in account['aliases']]
        to_remove_aliases = [x for x in account['aliases'] if x not in should_be_aliases]
        matched_aliases = [x for x in should_be_aliases if x in account['aliases']]

        if matched_aliases:
            logger.debug("Matched personal aliases: {}".format(matched_aliases))
        if to_add_aliases:
            logger.debug("Should be added to personal aliases: {}".format(to_add_aliases))
            changes.append(('extra_aliases', ["+{}".format(x) for x in to_add_aliases]))
        if to_remove_aliases:
            logger.debug("Should be removed from personal aliases: {}".format(to_remove_aliases))
            changes.append(('extra_aliases', ["-{}".format(x) for x in to_remove_aliases]))

        if to_add_aliases or to_remove_aliases:
            if fix:
                self.google.update_group_aliases(mp, to_add_aliases, to_remove_aliases)

        # Check permissions of a group.
        email = settings.CLAUDIA_GSUITE['DOMAIN_ADMIN_ACCOUNT_EMAIL']
        gsettings_api = GoogleSuiteAPI.create_directory_service(
            user_email=email,
            api_name="groupssettings",
            api_version="v1",
            scopes=settings.CLAUDIA_GSUITE['SCOPES']['GROUPSSETTINGS_API']
        )

        # Get group to generate changes list
        group = None
        try:
            group = gsettings_api.groups().get(groupUniqueId=mp.email).execute()
        except googleapiclient.errors.HttpError as e:
            logger.warning("Could not get settings info for group {} - Message: {}".format(mp, e))

        # Only allow members to view other members and past messages if the group is not open for signup
        # We also disable some other features in those cases; allowing posting as the group, etc.
        # Amelie issue #782 -- https://github.com/Inter-Actief/amelie/issues/782
        mp_obj = mp.get_mapped_object()
        if isinstance(mp_obj, AliasGroup) and mp_obj.open_to_signup:
            logger.debug("This is a public sign-up group, so some GSuite features will be restricted or disabled")
            who_can_view_membership = "ALL_MANAGERS_CAN_VIEW"  # Only managers can see the other members
            who_can_view_group = "ALL_MANAGERS_CAN_VIEW"  # Only managers can see the messages in the group
            allow_web_posting = "false"  # Posts via the web forum are not allowed
            members_can_post_as_group = "false"  # Do not allow sending messages using the group's address
            who_can_moderate_content = "OWNERS_AND_MANAGERS"  # Only owners and managers may moderate content (e.g. manage spam)
            who_can_assist_content = "OWNERS_AND_MANAGERS"  # Only owners and managers may manage content assistant controls
        else:
            who_can_view_membership = "ALL_MEMBERS_CAN_VIEW"  # Members can see the other members
            who_can_view_group = "ALL_MEMBERS_CAN_VIEW"  # Members can see the messages in the group
            allow_web_posting = "true"  # Posts via the web forum are allowed
            members_can_post_as_group = "true"  # Members are allowed to send messages using the group's address
            who_can_moderate_content = "ALL_MEMBERS"  # All members may moderate content (e.g. manage spam)
            who_can_assist_content = "ALL_MEMBERS"  # All members may manage content assistant controls

        default_settings = {
            "kind": "groupsSettings#groups",
            "whoCanJoin": "INVITED_CAN_JOIN",  # Only invited members may join
            "whoCanViewMembership": who_can_view_membership,  # See above
            "whoCanViewGroup": who_can_view_group,  # See above
            "allowExternalMembers": "false",  # No external members may view and join the group
            "whoCanPostMessage": "ANYONE_CAN_POST",  # Anyone may e-mail this group
            "allowWebPosting": allow_web_posting,  # See above
            "isArchived": "true",  # Messages sent to this group are kept in an archive
            "archiveOnly": "false",  # This group is not inactive
            "messageModerationLevel": "MODERATE_NONE",  # Messages are not moderated, and sent directly to the group
            "spamModerationLevel": "MODERATE",  # Send detected spam messages to the moderation queue, but don't notify anyone
            "replyTo": "REPLY_TO_IGNORE",  # Users may decide where the message reply is sent
            "includeCustomFooter": "false",  # No custom footer on each message
            "sendMessageDenyNotification": "false",  # Do not send a notification to authors if their message was rejected
            "membersCanPostAsTheGroup": members_can_post_as_group,  # See above
            "includeInGlobalAddressList": "true",  # This group is visible in the address book
            "whoCanLeaveGroup": "NONE_CAN_LEAVE",  # You cannot escape!
            "whoCanContactOwner": "ALL_MEMBERS_CAN_CONTACT",  # Members may contact the group owner (claudia)
            "whoCanModerateMembers": "NONE",  # No one may moderate members
            "whoCanModerateContent": who_can_moderate_content,  # See above
            "whoCanAssistContent": who_can_assist_content,  # See above
            "whoCanDiscoverGroup": "ALL_MEMBERS_CAN_DISCOVER"  # Only members may discover this group
        }

        if group:
            diff = {x: (group[x] if x in group.keys() else "", default_settings[x])
                    for x in default_settings.keys() if x not in group.keys() or group[x] != default_settings[x]}

            if len(diff) != 0:
                for x, y in diff.items():
                    changes.append(("group_settings_{}".format(x), y))

                try:
                    gsettings_api.groups().patch(
                        groupUniqueId=mp.email,
                        body=default_settings
                    ).execute()
                except googleapiclient.errors.HttpError as e:
                    logger.warning("Unable to update group settings for {} - Message: {}".format(mp, e))

        return changes

    def verify_mapping_shareddrive(self, claudia: Claudia, mp: Mapping, account, fix: bool = False) -> List[Tuple[str, Iterable[str]]]:
        changes = []

        logger.debug("Checking Shared Drive-specific details for {object_name} (cid: {cid})".format(
            object_name=mp.name, cid=mp.id
        ))

        # Check Shared Drive name
        if account['name'] != mp.name:
            logger.debug("Name changed to {} (was: {})".format(mp.name, account['name']))
            changes.append(("name", (account['name'], mp.name)))
            if fix:
                self.google.update_shared_drive(mp)

        # Get all mappings that should be in the drive
        should_be_in = mp.members()

        # Get all mappings that are currently in the drive
        currently_in = self.google.get_drive_permissions(mp)
        currently_in_ids = [x['id'] for x in currently_in['permissions']]
        currently_in_permissions = DrivePermission.objects.filter(drive=mp.get_mapped_object(),
                                                                  permission_id__in=currently_in_ids)
        missing_ids = [x for x in currently_in_ids if x not in [y.permission_id for y in currently_in_permissions]]
        currently_in_mappings = [x.mapping for x in currently_in_permissions]

        to_add_mappings = [x for x in should_be_in if x not in currently_in_mappings]
        to_remove_mappings = [x for x in currently_in_mappings if x not in should_be_in]
        matched_mappings = [x for x in should_be_in if x in currently_in_mappings]

        to_remove_ids = [
            x for x in missing_ids if not self.google.get_drive_permission(drive=mp, id=x)['emailAddress']
                                          == settings.CLAUDIA_GSUITE['DOMAIN_ADMIN_ACCOUNT_EMAIL']
        ]

        if matched_mappings:
            logger.debug("Matched mappings of shared drives: {}".format(matched_mappings))
        if to_add_mappings:
            logger.debug("Should be added to shared drive: {}".format(to_add_mappings))
            changes.append(('shared_drive', ["+{}".format(x) for x in to_add_mappings]))
            if fix:
                for x in to_add_mappings:
                    self.google.create_drive_permission(mp, x)
        if to_remove_mappings:
            logger.debug("Should be removed from shared drive: {}".format(to_remove_mappings))
            changes.append(('shared_drive', ["-{}".format(x) for x in to_remove_mappings]))
            if fix:
                for x in to_remove_mappings:
                    self.google.delete_drive_permission(mp, x)
        if to_remove_ids:
            logger.debug("Should be removed from shared drive: {}".format(to_remove_ids))
            changes.append(('shared_drive', ["-{}".format(x) for x in to_remove_ids]))
            if fix:
                for x in to_remove_ids:
                    self.google.delete_drive_permission_by_id(mp, x)

        return changes


from api.models import TaskList
from api.serializers import TaskListSerializer2, TasksSerializer
from rest_framework.response import Response
from rest_framework.decorators import api_view
from rest_framework import status

@api_view(['GET', 'POST'])
def task_list(request):
    if request.method == 'GET':
        task_lists = TaskList.objects.all()
        serializer = TaskListSerializer2(task_lists, many=True)
        return Response(serializer.data, status=status.HTTP_200_OK)
    elif request.method == 'POST':
        serializer = TaskListSerializer2(data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status=status.HTTP_201_CREATED)
        return Response(serializer.errors, status=status.HTTP_500_INTERNAL_SERVER_ERROR)

@api_view(['GET', 'PUT', 'DELETE'])
def task_detail(request, pk):
    try:
        task_list = TaskList.objects.get(id = pk)
    except TaskList.DoesNotExist:
        return Response(status=status.HTTP_404_NOT_FOUND)

    if request.method == 'GET':
        serializer = TaskListSerializer2(task_list)
        return Response(serializer.data)
    elif request.method == 'PUT':
        serializer = TaskListSerializer2(instance = task_list, data = request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data)
        return Response(serializer.errors)
    elif request.method == 'DELETE':
        task_list.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)

@api_view(['GET'])
def task_list_tasks(request, pk):
    try:
        task_list = TaskList.objects.get(id=pk)
    except TaskList.DoesNotExist as e:
        return Response({'error': str(e)})

    tasks = task_list.tasks.all()
    serializer = TasksSerializer(tasks, many=True)
    return Response(serializer.data, status=status.HTTP_200_OK)

@api_view(['GET', 'POST', 'DELETE', 'PUT'])
def task_list_tasks_detail(request, pk, ik):
    try:
        task_list = TaskList.objects.get(id=pk)
    except TaskList.DoesNotExist as e:
        return Response({'error': str(e)})

    if request.method == 'GET':
        tasks = task_list.task_set.all();
        for t in tasks:
            if t.id == pk:
                serializer = TasksSerializer(t)
                return Response(serializer.data, safe=False)

    elif request.method == 'POST':
        serializer = TasksSerializer(data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status=200)
        return Response(serializer.errors)

    elif request.method == 'DELETE':
        tasks = task_list.task_set.all();
        for t in tasks:
            if t.id == pk:
                t.delete()
                return Response({}, status=204)

    elif request.method == 'PUT':
        tasks = task_list.tasks.all()
        for t in tasks:
            if t.id == pk:
                serializer = TasksSerializer(instance=t, data=request.data)
                if serializer.is_valid():
                    serializer.save()  # update function in serializer class
                    return Response(serializer.data, status=200)
                return Response(serializer.errors)


#!/usr/bin/env python3
"""
CH21 - Errors and Debugging
@author Ellery Penas
@version 2018.04.18
"""

def colorize(text, color):
    colors = ('red', 'green', 'blue')
    if not isinstance(text, str):
        raise TypeError("Type of text must be an instance of str")
    if color not in colors:
        raise TypeError(
            f"Invalid color specified '{color}': valid colors are red, green or blue")
    print(f"printed {text} in the color {color}")

def get(d, key):
    """
    Will catch KeyError exception if key does not exist on dict
    """
    try:
        return d[key]
    except KeyError:
        return None

def main():
    """main line function"""
    # raise ValueError("blah blah")
    # colorize("Hello", "red")
    # colorize("Hello", "yellow")
    # colorize(10, "red")  # will raise TypeError

    # try - except exploration
    d = {"name": "Jeff"}
    print(get(d, "name"))
    print(get(d, "city"))  # raises KeyError as city does not exist

if __name__ == "__main__":
    main()


import sys
import pyspark.sql.functions as F
from pyspark.ml.evaluation import RegressionEvaluator
from pyspark.sql import Row
from pyspark.sql import SparkSession
from pyspark.sql.functions import udf
from pyspark.sql.functions import desc
import operator

spark = SparkSession.builder.appName("demo-cluster").getOrCreate()

# WRITE
def map_user_products_file(line):
	result = {}
	parts = line.value.split(';')
	for index in range(1, len(parts) - 1):
		key = [int(i) for i in parts[index].split(':')[0].split(',')]
		value = int(parts[index].split(':')[1])
		result[tuple(key)] = int(value)

	return (parts[0], result)

user_products_file = spark.read.text("tmp_output_file/user_product_matrix_237.ouput").rdd
user_product_matrix_rdd = user_products_file.map(map_user_products_file)

# GEN recommended cates
no_of_cates = 6

def map_recommended_cates(user):
	top_user_cates = sorted(user[1].iteritems(), key=operator.itemgetter(1), reverse=True)[:no_of_cates]

	user_recommended_cates = []

	for cate in top_user_cates:
		user_recommended_cates.append(cate[0])

	return (user[0], user_recommended_cates)

recommended_cates = user_product_matrix_rdd.map(map_recommended_cates).collect()
file = open("tmp_output_file/recommended_cates_237_6.ouput", 'w')
for row in recommended_cates:
	file.write(str(row[0]))
	file.write(';')

	for index1 in range(0, len(row[1])):
		key = list(row[1][index1])
		for index2 in range(0, len(key)):
			file.write(str(key[index2]))
			if index2 != len(key) - 1:
				file.write(',')
		if index1 != len(row[1]) - 1:
			file.write(';')
	file.write('\n')
file.close()

# READER
# def map_recommended_cates_file(line):
# 	parts = line.value.split(';')
# 	user_id = parts[0]
# 	cates = []
# 	for index in range(1, len(parts)):
# 		cates.append(tuple([int(x) for x in parts[index].split(',')]))
# 	return (user_id, cates)

# recommended_cates_file = spark.read.text("tmp_output_file/recommended_cates_237_5.ouput").rdd
# recommended_cates = recommended_cates_file.map(map_recommended_cates_file)
# recommended_cates_dict = recommended_cates.collectAsMap()
# print(recommended_cates_dict)


import numpy as np
from typing import Optional
from dataclasses import dataclass, field

@dataclass
class Config:
    q: np.ndarray
    index: int = field(default_factory=lambda :-1)

    def copy(self)->"Config":
        return Config(self.q)

class Tree:
    def __init__(self, root: Config):
        root.index = 0
        self.root = root
        self.data = [root]
        self.parent = {0:-1}
        self.num = 1

    def add_node(self, node: Config, parent: Config):
        assert node.index != parent.index
        node.index = self.num
        self.parent[node.index] = parent.index
        self.data.append(node)
        self.num += 1

    def nearest(self, node: Config):
        distances = []
        for node_tree in self.data:
            d = np.linalg.norm(node_tree.q - node.q)
            distances.append(d)
        idx = np.argmin(distances)
        return self.data[idx]

    def backtrack(self, node: Config):
        path = []
        node_curr = node
        while True:
            path.append(node_curr)
            parent_index = self.parent[node_curr.index]
            if parent_index == -1:
                break
            node_curr = self.data[parent_index]
        return path[::-1]


#! /usr/bin/python

__author__ = 'Kaushik Acharya'
__date__ = "$Jun 29, 2014"

from count_freqs import Hmm
import sys
from collections import defaultdict
from math import log
import re

def enum(**enums):
    # type creates a dynamic form of the class statement
    return type('Enum', (object,), enums)

class Emission:
    def __init__(self, original_train_file, modified_train_file, count_threshold=5, categorize_rare=False, epsilon=1e-07):
        # in modified_train_file we map infrequent words to a common class
        self.original_train_file = original_train_file
        self.modified_train_file = modified_train_file
        self.count_threshold = count_threshold
        self.emission_counts = defaultdict(int)
        self.ne_tags = []
        self.categorize_rare = categorize_rare
        self.rare_type = enum(NUMERIC=0, ALL_CAPS=1, LAST_CAPS=2, RARE=3)
        self.epsilon = epsilon

    def train_ngram_and_emission_freq_from_corpus_file(self, corpus_file):
        counter = Hmm(3)
        counter.train(corpus_file)
        self.emission_counts = counter.emission_counts
        self.ngram_counts = counter.ngram_counts

    def collect_ne_tags(self):
        for ngram in self.ngram_counts[0]:
            self.ne_tags.append(ngram[0])

    def is_rare_word(self, word):
        freq_count = 0
        for ne_tag in self.ne_tags:
            elem = (word, ne_tag)
            freq_count += self.emission_counts[elem]

        return freq_count < self.count_threshold

    def type_rare_word(self, word):
        # Note: the requirement of ^ and $
        # http://stackoverflow.com/questions/9999726/python-regular-expression-matching-either-all-uppercase-letter-and-number-or-jus
        # http://stackoverflow.com/questions/14471177/python-check-if-the-last-characters-in-a-string-are-numbers#
        if re.search(r'\d', word) is not None:
            return self.rare_type.NUMERIC
        elif re.match(r'^[A-Z]+$', word) is not None:
            return self.rare_type.ALL_CAPS
        elif re.search(r'[A-Z]+$', word) is not None:
            return self.rare_type.LAST_CAPS
        else:
            return self.rare_type.RARE

    def rare_word(self, word):
        if self.categorize_rare is False:
            return '_RARE_'
        else:
            type_rare = self.type_rare_word(word)
            if type_rare == self.rare_type.NUMERIC:
                return '_RARE_NUMERIC_'
            elif type_rare == self.rare_type.ALL_CAPS:
                return '_RARE_ALL_CAPS_'
            elif type_rare == self.rare_type.LAST_CAPS:
                return '_RARE_LAST_CAPS_'
            else:
                return '_RARE_'

    def assign_rare_symbol_to_infrequent_words(self):
        with open(self.modified_train_file, 'w') as wfd:
            with open(self.original_train_file, 'r') as rfd:
                for line in rfd:
                    tokens = line.split()
                    if len(tokens) == 0:
                        wfd.write('%s' % line)  # blank line
                        continue
                    word = tokens[0]
                    ne_tag = tokens[1]
                    if self.is_rare_word(word):
                        word = self.rare_word(word)
                    wfd.write('%s %s\n' % (word, ne_tag))

    def gene_tagger_using_emission_param(self, word):
        if self.is_rare_word(word):
            word = self.rare_word(word)

        max_emission_parameter = 0
        max_emission_tag = None
        for ne_tag in self.ne_tags:
            emission_parameter = self.compute_emission_parameter(word, ne_tag)
            if max_emission_parameter < emission_parameter:
                max_emission_parameter = emission_parameter
                max_emission_tag = ne_tag

        assert (max_emission_tag is not None), "max_emission_parameter not computed"
        return max_emission_tag

    def compute_emission_parameter(self, word, ne_tag):
        elem = (word, ne_tag)
        freq_emission = self.emission_counts[elem]
        freq_tag = self.ngram_counts[0][(ne_tag,)]

        return freq_emission*1.0/freq_tag

    def compute_trigram_transition_parameter(self, ngram):
        if self.ngram_counts[1][ngram[0:2]] == 0:
            return self.epsilon
        if self.ngram_counts[2][ngram] == 0:
            return self.epsilon
        return self.ngram_counts[2][ngram]*1.0/self.ngram_counts[1][ngram[0:2]]

    def assign_gene_tag_using_emission_param(self, dev_input_file, dev_output_file):
        with open(dev_output_file, 'w') as wfd:
            with open(dev_input_file, 'r') as rfd:
                for line in rfd:
                    tokens = line.split()
                    if len(tokens) == 0:
                        wfd.write('%s'%(line))  # blank line
                        continue
                    word = tokens[0]
                    ne_tag = self.gene_tagger_using_emission_param(word)
                    wfd.write('%s %s\n' % (word, ne_tag))

    def assign_gene_tag_using_viterbi(self, dev_input_file, dev_output_file):
        with open(dev_output_file, 'w') as wfd:
            with open(dev_input_file, 'r') as rfd:
                sentence_tokens = []
                for line in rfd:
                    tokens = line.split()
                    if len(tokens) == 0:
                        final_tag_sequence = self.compute_max_prob_tag_sequence_for_all_pos(sentence_tokens)
                        # print 'tag sequence: ', self.final_tag_sequence
                        for token_i in range(0, len(sentence_tokens)):
                            wfd.write('%s %s\n' % (sentence_tokens[token_i], final_tag_sequence[token_i]))
                        wfd.write('%s' % (line))  # blank line
                        sentence_tokens = []
                        continue
                    word = tokens[0]
                    sentence_tokens.append(word)

    def max_prob_tag_sequence_at_pos(self, k, u, v, word_at_pos_k):
        # returns log prob
        # values assigned in dynamic programming table
        # output: max prob of a tag sequence ending in tags u,v at position k  i.e. u at position (k-1) and v at pos k
        arg_max_prob_tag = str()
        if k in [1, 2]:
            # w belongs to {*}
            bigram = tuple(['*', u])
            max_prob_tag_seq_at_prev_pos = self.max_prob_tag_sequence[k-1][bigram]
            trigram = tuple(['*', u, v])
            trigram_transition_parameter = self.compute_trigram_transition_parameter(trigram)
            # max_prob_tag_seq = max_prob_tag_seq_at_prev_pos * trigram_transition_parameter
            max_prob_tag_seq = max_prob_tag_seq_at_prev_pos + log(trigram_transition_parameter)
        else:
            # w belongs to S
            max_prob_tag_seq = float('-inf')
            for w in self.ne_tags:
                bigram = tuple([w, u])
                max_prob_tag_seq_at_prev_pos = self.max_prob_tag_sequence[k-1][bigram]
                trigram = tuple([w, u, v])
                trigram_transition_parameter = self.compute_trigram_transition_parameter(trigram)
                # prob_tag_seq = max_prob_tag_seq_at_prev_pos * trigram_transition_parameter
                prob_tag_seq = max_prob_tag_seq_at_prev_pos + log(trigram_transition_parameter)
                if max_prob_tag_seq < prob_tag_seq:
                    max_prob_tag_seq = prob_tag_seq
                    arg_max_prob_tag = w

        # now multiply with emission parameter
        if self.is_rare_word(word_at_pos_k):
            word_at_pos_k = self.rare_word(word_at_pos_k)  # '_RARE_'
        # max_prob_tag_seq *= self.compute_emission_parameter(word_at_pos_k, v)
        prob_emission = self.compute_emission_parameter(word_at_pos_k, v)
        if prob_emission == 0:
            max_prob_tag_seq = float('-inf')
        else:
            max_prob_tag_seq += log(prob_emission)

        bigram = tuple([u, v])
        self.max_prob_tag_sequence[k][bigram] = max_prob_tag_seq
        self.back_pointer[k][bigram] = arg_max_prob_tag

    def compute_max_prob_tag_sequence_for_all_pos(self, sentence_tokens):
        # log prob
        self.max_prob_tag_sequence = [defaultdict(int) for i in xrange(len(sentence_tokens)+2)]
        self.back_pointer = [defaultdict(str) for i in xrange(len(sentence_tokens)+2)]
        # base case
        bigram = tuple(['*', '*'])
        self.max_prob_tag_sequence[0][bigram] = log(1.0)

        for k in range(1, len(sentence_tokens)+1):
            if k < 2:
                tag_list_for_pos_u = ['*']
            else:
                tag_list_for_pos_u = self.ne_tags

            # u belongs to S(k-1) and v belongs to S(k)
            for u in tag_list_for_pos_u:
                for v in self.ne_tags:
                    self.max_prob_tag_sequence_at_pos(k, u, v, sentence_tokens[k-1])

        # Now assign the tag sequence backwards
        max_prob = float('-inf')
        # Note: final_tag_sequence[0,1, ...,(n-1)] whereas
        #       self.max_prob_tag_sequence and self.back_pointer are 1-based indexed
        final_tag_sequence = [str() for i in xrange(len(sentence_tokens))]
        # print 'sentence_tokens: ', sentence_tokens
        # print 'length(final_tag_sequence): ', len(final_tag_sequence)
        for u in tag_list_for_pos_u:
            for v in self.ne_tags:
                bigram = tuple([u, v])
                trigram = tuple([u, v, 'STOP'])
                prob = self.max_prob_tag_sequence[len(sentence_tokens)][bigram] + \
                       log(self.compute_trigram_transition_parameter(trigram))
                if max_prob < prob:
                    final_tag_sequence[len(sentence_tokens)-2] = u
                    final_tag_sequence[len(sentence_tokens)-1] = v
                    max_prob = prob

        for k in range(len(sentence_tokens)-3, -1, -1):
            bigram = tuple([final_tag_sequence[k+1], final_tag_sequence[k+2]])
            final_tag_sequence[k] = self.back_pointer[k+3][bigram]

        return final_tag_sequence

if __name__ == "__main__":
    original_train_file = sys.argv[1]
    modified_train_file = sys.argv[2]
    count_threshold = int(sys.argv[3])
    dev_input_file = sys.argv[4]
    dev_output_file = sys.argv[5]
    part_no = int(sys.argv[6])

    if part_no == 3:
        categorize_rare_flag = True
    else:
        categorize_rare_flag = False

    emission = Emission(original_train_file, modified_train_file, count_threshold, categorize_rare_flag)
    with open(original_train_file, 'r') as fd:
        emission.train_ngram_and_emission_freq_from_corpus_file(fd)

    emission.collect_ne_tags()
    emission.assign_rare_symbol_to_infrequent_words()

    # re-calculate counts again
    with open(modified_train_file, 'r') as fd:
        emission.train_ngram_and_emission_freq_from_corpus_file(fd)

    if part_no == 1:
        emission.assign_gene_tag_using_emission_param(dev_input_file, dev_output_file)
    elif part_no in [2, 3]:
        emission.assign_gene_tag_using_viterbi(dev_input_file, dev_output_file)

'''
Need for log transform, though staff claims that's not mandatory.
https://class.coursera.org/nlangp-001/forum/thread?thread_id=505

http://stackoverflow.com/questions/15721363/preserve-python-tuples-with-json

Enums:
    http://www.pythonexamples.org/2011/01/12/how-to-create-an-enum-in-python/
    http://www.pythoncentral.io/how-to-implement-an-enum-in-python/

Regular expressions:
    http://www.tutorialspoint.com/python/python_reg_expressions.htm
'''

'''
    corpus_file = sys.argv[1]
    counter = Hmm(3)
    with open(corpus_file) as fd:
        counter.read_counts(fd)

    # test
    elem = ("Sls1p", "I-GENE")
    print "emission count: {0}".format(counter.emission_counts[elem])
    ngram = tuple(elem[1:])
    print "tag count: {0}".format(counter.ngram_counts[0][ngram])
    elem = ("Sls1p", "O")
    if elem in counter.emission_counts:
        print "present with another tag: {0}".format(elem[1])
    else:
        print "absent with another tag: {0}".format(elem[1])

    # printing all the symbols which have both the tag
    for elem in counter.emission_counts.keys():
        if elem[1] ==  "I-GENE":
            other_tag = "O"
        else:
            other_tag = "I-GENE"
        other_elem = (elem[0], other_tag)
        if other_elem in counter.emission_counts:
            print 'symbol {0} present with both tags'.format(elem[0])

    count_key = 0
    for elem in counter.emission_counts.keys():
        print '{0} : {1} : {2}'.format(count_key,elem[0],counter.emission_counts[elem])
        count_key += 1
        if count_key > 6:
            break
'''


#!/usr/bin/python
import pygame
import time

picture = pygame.image.load("nero.jpg") #Carica l'immagine
picture = pygame.transform.scale(picture,(1280,720))
panel = pygame.display.set_mode((1,1), pygame.FULLSCREEN)
pygame.mouse.set_visible(False) #Nasconde il cursore
panel.blit(picture,(0,0))
pygame.display.flip()
time.sleep(10000) #Timeout esagerato


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os.path
from conans import ConanFile, CMake
from conans.tools import Git
from conans.errors import ConanInvalidConfiguration

class ImguiSfmlConan(ConanFile):
    """ Imgui-SFML library conan file.

    Options:

    * shared: True or False
      Whether to build this library shared or static.

    * fPIC: True or False
      Whether or not to use -fPIC option while building.

    * imconfig: None or String
      Use 'None' if you want this library to provide default
      ImGui's imconfig.h file. Otherwise, provide a path to the
      imconfig.h file you want to include in the build.

    * imconfig_install_folder: None or String
      Use 'None' if you want this build to include custom config
      file, otherwise specify the directory where your imconfig
      will be installed.

    * imgui_revision: String
      Tag or branch of ImGui repository that should be used with
      ImGui-SFML.
    """

    name = 'imgui-sfml'
    version = '2.1'
    description = 'ImGui binding for use with SFML'
    topics = ('conan', 'sfml', 'gui', 'imgui')
    url = 'https://github.com/eliasdaler/imgui-sfml'
    homepage = 'https://github.com/eliasdaler/imgui-sfml'
    author = 'Elias Daler <eliasdaler@yandex.ru>'
    requires = 'sfml/2.5.1@bincrafters/stable'
    license = 'MIT'
    settings = 'os', 'compiler', 'build_type', 'arch'
    options = {
        'shared': [True, False],
        'fPIC': [True, False],
        'imconfig': 'ANY',
        'imconfig_install_folder': 'ANY',
        'imgui_revision': 'ANY'
    }
    default_options = {
        'shared': False,
        'fPIC': True,
        'imconfig': None,
        'imconfig_install_folder': None,
        'imgui_revision': 'v1.70'
    }
    exports_sources = [
        'cmake/FindImGui.cmake',
        'CMakeLists.txt',
        'imconfig-SFML.h',
        'imgui-SFML.cpp',
        'imgui-SFML_export.h',
        'imgui-SFML.h',
    ]
    exports = 'LICENSE'
    _imgui_dir = 'imgui'

    def source(self):
        git = Git(folder=self._imgui_dir)
        git.clone('https://github.com/ocornut/imgui.git', str(self.options.imgui_revision))

    def configure(self):
        imconfig = self.options.imconfig
        if imconfig:
            if not os.path.isfile(str(imconfig)):
                raise ConanInvalidConfiguration("Provided user config is not a file or doesn't exist")
            else:
                self._imconfig_path = os.path.abspath(str(self.options.imconfig))
        if not self.options.imgui_revision:
            raise ConanInvalidConfiguration("ImGui revision is empty. Try latest version tag or 'master'")

        self.options['sfml'].graphics = True
        self.options['sfml'].window = True
        self.options['sfml'].shared = self.options.shared

    def _configure_cmake(self):
        cmake = CMake(self)
        cmake.definitions['IMGUI_DIR'] = os.path.join(self.source_folder, self._imgui_dir)
        cmake.definitions['SFML_DIR'] = os.path.join(self.deps_cpp_info['sfml'].lib_paths[0], 'cmake', 'SFML')
        cmake.definitions['IMGUI_SFML_BUILD_EXAMPLES'] = 'OFF'
        cmake.definitions['IMGUI_SFML_FIND_SFML'] = 'ON'
        if self.options.imconfig_install_folder:
            cmake.definitions['IMGUI_SFML_CONFIG_INSTALL_DIR'] = self.options.imconfig_install_folder
        if self.options.imconfig:
            cmake.definitions['IMGUI_SFML_USE_DEFAULT_CONFIG'] = 'OFF'
            cmake.definitions['IMGUI_SFML_CONFIG_NAME'] = os.path.basename(self._imconfig_path)
            cmake.definitions['IMGUI_SFML_CONFIG_DIR'] = os.path.dirname(self._imconfig_path)
        else:
            cmake.definitions['IMGUI_SFML_USE_DEFAULT_CONFIG'] = 'ON'
        cmake.configure()
        return cmake

    def build(self):
        cmake = self._configure_cmake()
        cmake.build()

    def package(self):
        cmake = self._configure_cmake()
        cmake.install()

    def package_info(self):
        self.cpp_info.libs = ['ImGui-SFML']


import gzip
import numpy as np

def main():
    data = 0

    #gzipファイルからMNISTデータを呼び出して2次元の形にする
    with gzip.open('./train-images-idx3-ubyte.gz', 'rb') as f:
        data = np.frombuffer(f.read(), np.uint8, offset=16)
        data = data.reshape([-1,28,28])
    data = np.where(data < 102, -1, 1000)

    for e,imgtmp in enumerate(data):
        img = np.pad(imgtmp,[(2,2),(2,2)],"constant",constant_values=(-1))
        cnt = 0

        for i in range(2,30):
            for j in range(2,30):
                if img[i][j] == 1000:
                    img[i][j] = cnt
                    cnt+=1

        edges = []
        # y座標、x座標
        npzahyou = np.zeros((cnt,2))

        for i in range(2,30):
            for j in range(2,30):
                if img[i][j] == -1:
                    continue

                #8近傍に該当する部分を抜き取る。
                filter = img[i-2:i+3,j-2:j+3].flatten()
                filter1 = filter[[6,7,8,11,13,16,17,18]]

                npzahyou[filter[12]][0] = i-2
                npzahyou[filter[12]][1] = j-2

                for tmp in filter1:
                    if not tmp == -1:
                        edges.append([filter[12],tmp])

        np.save("../dataset/graphs/"+str(e),edges)
        np.save("../dataset/node_features/"+str(e),npzahyou)

if __name__=="__main__":
    main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = 'qxr'

import os
import os.path

def create_bg(filename):
    fid = open('/home/xiaoran/桌面/cascade_train/bg.txt', 'w')
    with open(filename) as f:
        lines = f.readlines()
        idx = 0
        while idx < len(lines):
            image_name = lines[idx].strip()
            image_name = os.path.join('/home/qxr/program/cascade_train/train_img_set/non-face', image_name)
            fid.write(image_name + '\n')
            idx += 1
        assert (idx == len(lines))
    fid.close()

if __name__ == '__main__':
    filename = '/home/xiaoran/桌面/cascade_train/neglist.txt'
    create_bg(filename)


import torch
import torch.nn as nn
import utils
from torch.autograd import Variable
import numpy as np
import torch.nn.functional as F

def get_feature_loss(args,embeded_feature,act_seq,vid_label):
        act_seq_diff=act_seq[:,1:]-act_seq[:,:-1]
        loss_feat=0
        for b in range(act_seq.shape[0]):
            gt_class=torch.nonzero(vid_label[b]).squeeze(1)
            loss_feat_batch=0
            act_count=0

            for c in gt_class:
                range_idx=torch.nonzero(act_seq_diff[b,:,c]).squeeze(1)
                range_idx=range_idx.cpu().data.numpy().tolist()
                if type(range_idx) is not list:
                    range_idx = [range_idx]
                if len(range_idx) == 0:
                    continue
                if act_seq_diff[b, range_idx[0], c] != 1:
                    range_idx = [-1] + range_idx
                if act_seq_diff[b, range_idx[-1], c] != -1:
                    range_idx = range_idx + [act_seq_diff.shape[1] - 1]

                label_lst = []
                feature_lsts = []

                # bkg
                if range_idx[0]>-1:
                    start_bkg=0
                    end_bkg=range_idx[0]
                    bkg_len=end_bkg-start_bkg+1
                    label_lst.append(0)
                    feature_lsts.append(utils.feature_sampling(args,embeded_feature[b],start_bkg, end_bkg + 1))

                # act
                for i in range(len(range_idx)//2):
                    if range_idx[2*i+1]-range_idx[2*i]<1:
                        continue
                    label_lst.append(1)
                    feature_lsts.append(utils.feature_sampling(args,embeded_feature[b],range_idx[2*i] + 1, range_idx[2*i + 1] + 1))

                    if range_idx[2*i+1]!=act_seq_diff.shape[1]-1:
                        start_bkg=range_idx[2*i + 1] + 1
                        if i == len(range_idx)//2-1:#act bkg -
                            end_bkg=act_seq_diff.shape[1]-1
                        else:#act bkg act
                            end_bkg=range_idx[2*i+2]
                        label_lst.append(0)
                        feature_lsts.append(utils.feature_sampling(args,embeded_feature[b],start_bkg,end_bkg+1))
                    act_count+=1

                if sum(label_lst)>1:
                    # for n in range(3):
                    #     feature_lst=[f[n] for f in feature_lsts]
                    feature_lst = torch.stack(feature_lsts, 0).clone()
                    feature_lst = feature_lst / torch.norm(feature_lst, dim=1, p=2).unsqueeze(1)
                    label_lst = torch.tensor(label_lst).to(args.device).float()
                    sim_matrix = torch.matmul(feature_lst, torch.transpose(feature_lst, 0, 1)) / 0.1
                    sim_matrix = torch.exp(sim_matrix)
                    sim_matrix = sim_matrix.clone().fill_diagonal_(0)
                    scores = (sim_matrix * label_lst.unsqueeze(1)).sum(dim=0) / sim_matrix.sum(dim=0)
                    loss_feat_batch += (-label_lst * torch.log(scores)).sum() / label_lst.sum()

            if act_count>0:
                loss_feat+=loss_feat_batch

        loss_feat=loss_feat/act_seq.shape[0]
        return loss_feat

def COUNTINGLOSS(features, gt_count, seq_len, device):
    ''' features: torch tensor dimension (B, n_element, n_class),
        gt_count: torch tensor dimension (B, n_class) of integer value,
        seq_len: numpy array of dimension (B,) indicating the length of each video in the batch,
        return: torch tensor of dimension 0 (value) '''

    pos_loss, neg_loss, num = 0, 0, 0
    inv_gt_count = (gt_count > 0).float() / (gt_count + 1e-10)
    for i in range(features.size(0)):
        # categories present in video
        mask_pos = (gt_count[i]<int(seq_len[i])) * (gt_count[i]>0)
        # categories absent
        mask_neg = (mask_pos==0)
        pred_count = (features[i,:seq_len[i]]).sum(0)
        pos_loss += ((pred_count[mask_pos] - Variable(gt_count[i][mask_pos],requires_grad=False)) * inv_gt_count[i][mask_pos]).abs().sum() # relative L1
        neg_loss += 0.001* pred_count[mask_neg==1].abs().sum()
        num += 1
    if num > 0:
        return (pos_loss+neg_loss)/num
    else:
        return torch.zeros(1).to(device)

def get_feature(seq,embeded_feature):#seq:[T]
    seq_diff=seq[1:]-seq[:-1]
    range_idx=torch.nonzero(seq_diff).squeeze(1)
    range_idx=range_idx.cpu().data.numpy().tolist()
    if type(range_idx) is not list:
        range_idx = [range_idx]
    if len(range_idx) == 0:
        return
    if seq_diff[range_idx[0]] != 1:
        range_idx = [-1] + range_idx
    if seq_diff[range_idx[-1]] != -1:
        range_idx = range_idx + [seq_diff.shape[0] - 1]

    feature_lsts = []
    idx=[]
    for i in range(len(range_idx)//2):
        if range_idx[2*i+1]-range_idx[2*i]<1:
            continue
        feature_lsts.append(embeded_feature[range_idx[2*i] + 1: range_idx[2*i + 1] + 1].clone())
        idx.append([range_idx[2*i] + 1,range_idx[2*i + 1] + 1])

    return feature_lsts

def KL_loss(pred,target,vid_label):
    kl_loss_total=0

    for b in range(pred.shape[0]):
        # kl_loss_func = nn.KLDivLoss(reduction="sum")#batchmean

        # pred_bkg_score=pred[b,:,-1]
        # tgt_bkg_score=target[b,:,-1]
        # bkg_kl_loss=kl_loss_func(F.log_softmax(pred_bkg_score,dim=0),F.softmax(tgt_bkg_score,dim=0))
        # # bkg_kl_loss2=kl_loss_func(pred_bkg_score,tgt_bkg_score)
        # act_kl_loss=kl_loss_func(F.log_softmax(pred[b],dim=0),F.softmax(target[b],dim=0))

        ce_criterion = nn.BCELoss(reduction='none')
        ce_loss=ce_criterion(pred[b],target[b].detach()).mean()

        kl_loss_total+=ce_loss

    kl_loss=kl_loss_total/pred.shape[0]

    return kl_loss

def contrastive_loss(args,proto,embeded_feature,act_seed,bkg_seed,vid_label): #proto:[C,N,D]
    loss_contra=0
    proto_vectors=proto.proto_vectors.clone().to(embeded_feature.device)
    proto_vectors_norm=proto_vectors/torch.norm(proto_vectors,dim=2,p=2).unsqueeze(2)

    intra_sim_matrix=torch.matmul(proto_vectors_norm,torch.transpose(proto_vectors_norm,1,2))
    loss_intra_proto=intra_sim_matrix.mean()

    for b in range(act_seed.shape[0]):
        gt_class=torch.nonzero(vid_label[b]).squeeze(1)
        act_featlst=[]
        for c in gt_class:
            act_featlst.append(get_feature(act_seed[b,:,c],embeded_feature[b,:,:].clone()))

        #bkg_act contra
        bkg_feat=get_feature(bkg_seed[b].squeeze(-1),embeded_feature[b,:,:].clone())
        if len(bkg_feat)==0:
            continue
        b_feat=torch.cat(bkg_feat,0).clone()
        b_feat = b_feat / torch.norm(b_feat, dim=1, p=2).unsqueeze(1)
        b_sim_matrix=torch.matmul(b_feat.unsqueeze(0).expand(args.action_cls_num,-1,-1), torch.transpose(proto_vectors_norm, 1, 2)) / 0.1
        b_sim_matrix = torch.exp(b_sim_matrix)

        #video_level contra
        for c_id,c_feat_lst in enumerate(act_featlst):
            if c_feat_lst is not None:
                if len(c_feat_lst)>0:
                    c=gt_class[c_id]
                    c_feat=torch.cat(c_feat_lst,0).clone()
                    c_feat = c_feat / torch.norm(c_feat, dim=1, p=2).unsqueeze(1)

                    a_sim_matrix = torch.matmul(c_feat.unsqueeze(0).expand(args.action_cls_num,-1,-1), torch.transpose(proto_vectors_norm, 1, 2)) / 0.1
                    a_sim_matrix = torch.exp(a_sim_matrix)
                    loss_contra_proto =  - torch.log(a_sim_matrix[c].sum().sum()/a_sim_matrix.sum().sum().sum())
                    # loss_contra_proto =  - torch.log(a_sim_matrix[c].max(dim=-1)[0].sum()/a_sim_matrix.sum())

                    loss_contra_bkg =  - torch.log(a_sim_matrix[c].mean().sum()/(a_sim_matrix[c].mean().sum()+b_sim_matrix[c].mean().sum()))

                    lambda_bkg=0.5
                    loss_contra += (1-lambda_bkg) * loss_contra_proto + lambda_bkg * loss_contra_bkg

                    # temporal structure loss
                    # L=5
                    # if len(c_feat_lst)>1:
                    #     sampled_feat_lst=[]
                    #     for inst_feat in c_feat_lst:
                    #         sampled_feat_lst.append(utils.feature_sampling(args,inst_feat,0,inst_feat.shape[0],L,'mean'))

                    #     sampled_feat_total=torch.cat(sampled_feat_lst,0).clone()
                    #     sampled_feat_total = sampled_feat_total / torch.norm(sampled_feat_total, dim=1, p=2).unsqueeze(1)
                    #     c_sim_matrix=torch.matmul(sampled_feat_total,torch.transpose(proto_vectors_norm[c],0,1))
                    #     match_score=F.softmax(c_sim_matrix*10,dim=1)  #temperture

                    #     num_pair=0
                    #     for inst_id in range(len(sampled_feat_lst)):
                    #         for inst_jd in range(inst_id+1,len(sampled_feat_lst)):
                    #             score_distance=torch.norm(match_score[inst_id*L:(inst_id+1)*L]-match_score[inst_jd*L:(inst_jd+1)*L],2,dim=1)
                    #             loss_temporal += score_distance.mean()

                    #             num_pair += 1
                    #     loss_temporal = loss_temporal / num_pair

        loss_contra=loss_contra/gt_class.shape[0]

    loss_contra=loss_contra/act_seed.shape[0]

    return loss_contra,loss_intra_proto

def base_loss(args,act_seed,bkg_seed,vid_score,vid_label,cas_sigmoid_fuse,point_anno):
    loss_frame=0
    loss_frame_bkg=0
    #video-level loss
    ce_criterion = nn.BCELoss(reduction='none')
    loss_vid = ce_criterion(vid_score, vid_label)
    loss_vid = loss_vid.mean()

    #frame-level loss
    #gt frame loss]
    act_seed=act_seed.to(args.device)
    focal_weight_act = (1 - cas_sigmoid_fuse) * point_anno + cas_sigmoid_fuse * (1 - point_anno)
    focal_weight_act = focal_weight_act ** 2
    weighting_seq_act = point_anno.max(dim=2, keepdim=True)[0]
    num_actions = point_anno.max(dim=2)[0].sum(dim=1)
    if num_actions>0:
        loss_frame = (((focal_weight_act * ce_criterion(cas_sigmoid_fuse, point_anno) * weighting_seq_act).sum(dim=2)).sum(dim=1) / num_actions).mean()

    #bkg frame loss
    bkg_seed = bkg_seed.unsqueeze(-1).to(args.device)
    point_anno_bkg = torch.zeros_like(point_anno).to(args.device)
    point_anno_bkg[:,:,-1] = 1

    weighting_seq_bkg = bkg_seed
    num_bkg = bkg_seed.sum(dim=1)
    focal_weight_bkg = (1 - cas_sigmoid_fuse) * point_anno_bkg + cas_sigmoid_fuse * (1 - point_anno_bkg)
    focal_weight_bkg = focal_weight_bkg ** 2
    if num_bkg>0:
        loss_frame_bkg = (((focal_weight_bkg * ce_criterion(cas_sigmoid_fuse, point_anno_bkg) * weighting_seq_bkg).sum(dim=2)).sum(dim=1) / num_bkg).mean()

    # #pseudo frame loss
    # act_seed = torch.cat((act_seed, torch.zeros((act_seed.shape[0], act_seed.shape[1], 1)).to(args.device)), dim=2)
    # focal_weight_pse_act = (1 - cas_sigmoid_fuse) * act_seed + cas_sigmoid_fuse * (1 - act_seed)
    # focal_weight_pse_act = focal_weight_pse_act ** 2
    # weighting_seq_pse_act = act_seed.max(dim=2, keepdim=True)[0].to(args.device)-weighting_seq_act
    # num_pse_actions=act_seed.max(dim=2)[0].sum(dim=1)-num_actions
    # if num_pse_actions>0:
    #     loss_pseudo_frame = (((focal_weight_pse_act * ce_criterion(cas_sigmoid_fuse, act_seed) * weighting_seq_pse_act).sum(dim=2)).sum(dim=1) / num_pse_actions).mean()
    #     loss_frame+=loss_pseudo_frame

    # #context_frame_loss
    # con_seed=con_seed.to(args.device)
    # focal_weight_con=(1-cas_sigmoid_fuse)*con_seed+cas_sigmoid_fuse*(1-con_seed)
    # focal_weight_con=focal_weight_con**2
    # weighting_seq_con = con_seed.max(dim=2, keepdim=True)[0].to(args.device)

    # num_con=con_seed.max(dim=2)[0].sum(dim=1)
    # if num_con>0:
    #     loss_con_frame=(((focal_weight_con * self.ce_criterion(cas_sigmoid_fuse, con_seed) * weighting_seq_con).sum(dim=2)).sum(dim=1) / num_con).mean()

    # if step%10==0:
    #     print('numbers: gt:{} pse_act:{} context:{} bkg:{}'.format(num_actions.cpu().item(),num_pse_actions.cpu().item(),num_con.cpu().item(),num_bkg.cpu().item()))

    return loss_vid,loss_frame,loss_frame_bkg

class Total_loss(nn.Module):
    def __init__(self,args):
        super(Total_loss, self).__init__()
        self.lambdas=args.lambdas
        self.ce_criterion = nn.BCELoss(reduction='none')

    def forward(self,args,step,vid_label,point_anno,proto,
                vid_score,cas_sigmoid_fuse,embeded_feature,
                vid_score_comp,embeded_feature_comp,cas_sigmoid_fuse_comp):
        loss={}

        #----------------------------------------base_loss---------------------------------------------------#
        loss_frame=0
        loss_frame_bkg=0
        loss_pseudo_frame=0

        point_anno = torch.cat((point_anno, torch.zeros((point_anno.shape[0], point_anno.shape[1], 1)).to(args.device)), dim=2)
        #select seed
        act_seed, bkg_seed, con_seed = utils.select_seed(cas_sigmoid_fuse.detach().cpu(),point_anno.detach().cpu(),step)
        # bkg_seed=pseudo_label['bkg_seq']
        # act_seed=pseudo_label['act_seq']

        loss_vid,loss_frame,loss_frame_bkg=base_loss(args,act_seed,bkg_seed,vid_score,vid_label,cas_sigmoid_fuse,point_anno)
        loss["loss_vid"]=loss_vid
        loss["loss_frame"]=loss_frame
        loss["loss_frame_bkg"]=loss_frame_bkg
        loss["loss_pseudo_frame"]=loss_pseudo_frame

        #----------------------------------------contrastive_loss---------------------------------------------------#
        loss_contrastive=0
        if bkg_seed.sum(dim=1)>0:
            loss_contrastive,loss_intra_proto=contrastive_loss(args,proto,embeded_feature,act_seed,bkg_seed,vid_label)
            loss["loss_contrastive"]=loss_contrastive
            loss["loss_intra_proto"]=loss_intra_proto
            proto.update_prototype(args,embeded_feature.detach(),act_seed,vid_label)

        #----------------------------------------kl_loss---------------------------------------------------#
        loss_kl=0
        # loss_kl=KL_loss(cas_sigmoid_fuse,cas_sigmoid_fuse_comp,vid_label)
        # loss['kl_loss']=loss_kl

        #----------------------------------------loss_count---------------------------------------------------#
        loss_count=0
        # countloss_mult = 1 if args.dataset=='THUMOS14' else 0.1
        # seq_len = torch.tensor(count_feat.shape[1]*torch.ones(count_feat.shape[0]),dtype=torch.int64)
        # if args.count_loss:
        #     loss_count=COUNTINGLOSS(count_feat, gt_count, seq_len, args.device)*countloss_mult
        #     loss["loss_count"]=loss_count

        #----------------------------------------loss_feat---------------------------------------------------#
        loss_feat=0
        # loss_feat=get_feature_loss(args,embeded_feature,act_seed,vid_label)
        # loss["loss_feat"]=loss_feat

        loss_total=self.lambdas[0]*loss_vid+self.lambdas[1]*loss_frame+self.lambdas[2]*loss_frame_bkg+self.lambdas[3]*loss_contrastive+\
                        1.0*loss_kl+1.0*loss_feat+0.5*loss_pseudo_frame
        loss["loss_total"]=loss_total

        return loss_total, loss

def train(net,proto,args,loader_iter,optimizer,scheduler,logger,step):
        net.train()
        total_loss={}
        total_cost=[]
        optimizer.zero_grad()

        for batch in range(args.batch_size):

            _idx, data, vid_label, point_anno, mask_seq, gt_heatmap, _video_name, _vid_len, _vid_duration=next(loader_iter)

            data=data.to(args.device)
            vid_label=vid_label.to(args.device)
            point_anno=point_anno.to(args.device)
            gt_heatmap=gt_heatmap.to(args.device)

            # attn_mask=pseudo_label['act_attn_mask']
            # if len(attn_mask.shape)>1:
            #     attn_mask=attn_mask.to(args.device)
            #     vid_score,cas_sigmoid,cas_sigmoid_fuse,embeded_feature,count_feat=net(data,proto.proto_vectors,vid_label,attn_mask)
            # else:
            #     vid_score,cas_sigmoid,cas_sigmoid_fuse,embeded_feature,count_feat=net(data, proto.proto_vectors, vid_label)
            # criterion=Total_loss(args)
            # cost,loss=criterion(args,step,vid_score,vid_label,cas_sigmoid,cas_sigmoid_fuse,point_anno,
            #                         pseudo_label,embeded_feature,count_feat,gt_count,proto)
            # total_cost.append(cost)

            vid_score_base,embeded_feature_base,cas_sigmoid_fuse_base,\
                vid_score_comp,embeded_feature_comp,cas_sigmoid_fuse_comp=net(data, proto.proto_vectors, vid_label, None)
            criterion=Total_loss(args)
            cost,loss=criterion(args,step,vid_label,point_anno,proto,
                                vid_score_base,cas_sigmoid_fuse_base,embeded_feature_base,
                                vid_score_comp,embeded_feature_comp,cas_sigmoid_fuse_comp)
            total_cost.append(cost)

            if not torch.isnan(cost):
                for key in loss.keys():
                    if not (key in total_loss.keys()):
                        total_loss[key]=[]

                    if loss[key]>0:
                        total_loss[key]+=[loss[key].detach().cpu().item()]
                    else:
                        total_loss[key] += [loss[key]]

        total_cost=sum(total_cost) / args.batch_size
        total_cost.backward(retain_graph=True) #retain_graph=True for contrastive loss
        optimizer.step()

        for key in total_loss.keys():
            # print("loss/{}:{}".format(key,sum(total_loss[key]) / args.batch_size))
            logger.log_value("loss/"+key,sum(total_loss[key]) / args.batch_size, step)

        if scheduler is not None:
            scheduler.step()
            lr = scheduler.get_last_lr()[0]
            logger.log_value("lr",lr, step)

def heatmap_loss(args,p_heatmap,gt_heatmap):
    ce_criterion = nn.BCELoss(reduction='none')
    a=ce_criterion(p_heatmap,gt_heatmap)
    mask_neg=torch.zeros((gt_heatmap.shape[0],gt_heatmap.shape[1],1)).to(args.device)
    mask_neg[gt_heatmap<0.1]=1
    num_neg=mask_neg.sum()
    mask_pos=1-mask_neg
    num_pos=mask_pos.sum()
    loss_heatmap_neg=((a*mask_neg).sum(dim=2).sum(dim=1)/num_neg).mean()
    loss_heatmap_pos=((a*mask_pos).sum(dim=2).sum(dim=1)/num_pos).mean()
    loss_heatmap=loss_heatmap_neg+loss_heatmap_pos
    # loss_norm=torch.norm(p_heatmap,p=2)
    # loss_heatmap+=0.1*loss_norm
    return loss_heatmap

def train_pgm(net,args,loader,optimizer):
    net.train()
    total_cost=[]
    optimizer.zero_grad()
    for batch in range(args.batch_size):
        _idx, data, vid_label, point_anno, pseudo_label, gt_heatmap, _video_name, _vid_len, _vid_duration=next(loader)
        data=data.to(args.device)
        gt_heatmap=gt_heatmap.to(args.device)
        p_heatmap=net(data)
        loss_heatmap=heatmap_loss(args,p_heatmap,gt_heatmap)
        total_cost.append(loss_heatmap)
    total_cost=sum(total_cost) / args.batch_size
    total_cost.backward()
    optimizer.step()
    return total_cost


class Node:
	def __init__(self, data=None, parent=None, depth=0, weight=0):
		self.data = data
		self.parent = parent
		self.weight = weight

	def findEmpty(self):
		for i in range(len(self.data)):
			if self.data[i] == 0:
				return i

	def up(self, index):
		arr = list(self.data)
		if index > 2:
			arr[index], arr[index-3] = arr[index-3], arr[index]
			return Node(data=arr, parent=self)
		else:
			return None

	def down(self, index):
		arr = list(self.data)
		if index < 6:
			arr[index], arr[index+3] = arr[index+3], arr[index]
			return Node(data=arr, parent=self)
		else:
			return None

	def left(self, index):
		arr = list(self.data)
		if index % 3 != 0:
			arr[index], arr[index-1] = arr[index-1], arr[index]
			return Node(data=arr, parent=self)
		else:
			return None

	def right(self, index):
		arr = list(self.data)
		if (index + 1) % 3 != 0:
			arr[index], arr[index+1] = arr[index+1], arr[index]
			return Node(data=arr, parent=self)
		else:
			return None

	def checkMoves(self):
		index = self.findEmpty()
		neighbors = []
		neighbors.append(self.up(index))
		neighbors.append(self.down(index))
		neighbors.append(self.left(index))
		neighbors.append(self.right(index))

		return [i for i in neighbors if i]


# niconico.py - Cookies

from typing import Optional

from http.cookies import SimpleCookie

from datetime import datetime, timedelta
from time import time

__all__ = ("Cookies",)
FORMAT = "%a, %d-%b-%Y %X"

class Cookies(SimpleCookie):
    @staticmethod
    def _make_cookie_tuple(name: str, cookie: str, domain: Optional[str] = ".nicovideo.jp"):
        cookies = Cookies()
        cookies[name] = cookie
        for key, value in (
            ("domain", domain), ("path", "/"),
            ("expires", (datetime.now() + timedelta(days=365)).strftime(FORMAT))
        ):
            cookies[name][key] = value
        return cookies

    @classmethod
    def from_file(cls, path: str):
        """Netscapeのクッキーファイルフォーマットに準拠したテキストファイルからクッキーが格納されたクラスを作成します。
        なので、ChromeやEdgeで使用可能な `Get cookies.txt <https://chrome.google.com/webstore/detail/get-cookiestxt/bgaddhkoddajcdgocldbbfleckgcbcid>`_ のような拡張機能を使って作ったクッキーのテキストファイルを読み込むことができます。
        もし自分のアカウントでニコニコ動画のコンテンツを取得したい場合はこれを使用しましょう。

        Parameters
        ----------
        path : str
            クッキーのファイルのパスです。

        Raises
        ------
        FileNotFoundError"""
        with open(path, "r") as f:
            raw = f.read()
        cookies = cls()
        for item in map(
            lambda x: x.split(), filter(
                lambda x: x and x[0] != "#", raw.splitlines()
            )
        ):
            cookies[item[5]] = item[6]
            for index, key in enumerate(
                ("domain", None, "path", "secure", "expires")
            ):
                if key:
                    cookies[item[5]][key] = (
                        datetime.fromtimestamp(float(item[index])).strftime(FORMAT)
                        if key == "expires" else item[index]
                    )
        return cookies

    @classmethod
    def from_string(cls, user_session: str):
        """ニコニコ動画上での認証済みのクッキーの値を直接指定しクラスを作成します。

        Parameters
        ----------
        user_session : str
            ユーザーセッションです。"""
        return cls._make_cookie_tuple("user_session", user_session)

    @classmethod
    def guest(cls, nicosid: Optional[str] = None):
        """ニコニコのゲストアカウントのクッキーを生成します。

        Notes
        -----
        これはニコニコ動画にアクセスして作成するクッキーではなく、開発者がクッキーを見て予想して作った再現物です。"""
        return cls._make_cookie_tuple("nicosid", nicosid or str(time()))


# page start
import math
print()

# functions
# varables
num_items = 0
box_items = 0
num_boxs = 0

# main code
num_items = int(input("Enter the number of items: "))
box_items = int(input("Enter the number of items per box: "))

num_boxs = num_items / box_items
num_boxs = math.ceil(num_boxs)

print(f"\nFor {num_items} items, packing {box_items} items in each box, you will need {num_boxs} boxes.")

# page end
print()


import numpy as np
from Old import data_in_out as io
import torch

class custom_loss(torch.autograd.function):

    @staticmethod
    def forward(ctx, distances, labels):
        ctx.save_for_backward(distances)
        same = torch.from_numpy(labels[:, None] == labels[:, None].transpose()).type(torch.LongTensor)
        same_distances = torch.sum(distances[same])

def train_step(features, distance, training_index, g_t):
    distances = torch.zeros((training_index.shape[0], training_index.shape[0]))
    labels = g_t[training_index]

    for i in range(training_index.shape[0]):
        for j in range(i+1):
            distances[i, j] = distance(features[:, i], features[:, j])

    constraint = torch.distributions.constraints.positive_definite
    constraint.check(distance)

# def minkowski_metric(x, y, p):
#
#     distances = - (y - x[:, None])
#     distances = (distances ** p)
#     distances = np.sum(distances, axis=0)
#     # distances = (distances ** 1/p)
#
#     return distances

# def KNN_classifier(features, gallery_indices, query_indices, gallery_mask):
#
#     features_classify = features[:, gallery_indices]
#     features_query = features[:, query_indices]
#     query_distances = np.zeros((query_indices.shape[0], gallery_indices.shape[0]))
#
#     for i in range(query_indices.shape[0]):
#         print('HERE: ', i)
#         gallery_mask_temp = np.repeat(gallery_mask[i, None], features.shape[0], axis=0)
#         query_distances[i, :] = minkowski_metric(features_query[:, i], np.ma.masked_where(gallery_mask_temp, features_classify), 2)
#     query_distances = np.ma.masked_where(gallery_mask, query_distances)
#
#     return query_distances

# def KNN_classifier(features, gallery_indices, query_indices, gallery_mask):
#
#     gallery_mindices = np.repeat(gallery_indices[None, :], query_indices.shape[0], axis=0)
#     gallery_mindices = np.ma.masked_where(gallery_mask, gallery_mindices)
#
#     features_classify = features[gallery_indices]
#     features_classify = np.repeat(features_classify.transpose()[None, ...], query_indices.shape[0], axis=0)
#     features_classify = features_classify[gallery_mindices[..., None]]
#
#     #labels_classify = labels[gallery_indices]
#     features_query = features[query_indices]
#     print(features_query.shape, features_classify.shape)
#
#     #distance = minkowski_metric(features_query, features_classify, 2)
#     return None

if __name__ == '__main__':

    features = np.memmap('PR_data/features', mode='r', shape=(14096, 2048), dtype=np.float64)
    features = torch.from_array(features.transpose())

    # data = loadmat('PR_data/cuhk03_new_protocol_config_labeled.mat')
    cam_ids = io.get_cam_ids()
    query_indices = io.get_query_indexes()
    gallery_indices = io.get_gallery_indexes()
    g_ts = io.get_ground_truth()

    # gallery_mask = eval.get_to_remove_mask(cam_ids, query_indices, gallery_indices, g_ts)
    # distances_query = KNN_classifier(features, gallery_indices, query_indices, gallery_mask)
    # print(distances_query.shape)


import sys
_module = sys.modules[__name__]
del sys
dataset = _module
main = _module
resnext = _module
sknet = _module
train = _module

from _paritybench_helpers import _mock_config, patch_functional
from unittest.mock import mock_open, MagicMock
from torch.autograd import Function
from torch.nn import Module
import abc, collections, copy, enum, functools, inspect, itertools, logging, math, matplotlib, numbers, numpy, pandas, queue, random, re, scipy, sklearn, string, tensorflow, time, torch, torchaudio, torchtext, torchvision, types, typing, uuid, warnings
import numpy as np
from torch import Tensor
patch_functional()
open = mock_open()
yaml = logging = sys = argparse = MagicMock()
ArgumentParser = argparse.ArgumentParser
_global_config = args = argv = cfg = config = params = _mock_config()
argparse.ArgumentParser.return_value.parse_args.return_value = _global_config
yaml.load.return_value = _global_config
sys.argv = _global_config
__version__ = '1.0.0'
xrange = range
wraps = functools.wraps

import numpy as np

from torch.utils.data import Dataset

import matplotlib.pyplot as plt

import torch

from torch import nn

from torch import optim

class ResNeXtUnit(nn.Module):

    def __init__(self, in_features, out_features, mid_features=None, stride=1, groups=32):
        super(ResNeXtUnit, self).__init__()
        if mid_features is None:
            mid_features = int(out_features / 2)
        self.feas = nn.Sequential(nn.Conv2d(in_features, mid_features, 1, stride=1), nn.BatchNorm2d(mid_features), nn.Conv2d(mid_features, mid_features, 3, stride=stride, padding=1, groups=groups), nn.BatchNorm2d(mid_features), nn.Conv2d(mid_features, out_features, 1, stride=1), nn.BatchNorm2d(out_features))
        if in_features == out_features:
            self.shortcut = nn.Sequential()
        else:
            self.shortcut = nn.Sequential(nn.Conv2d(in_features, out_features, 1, stride=stride), nn.BatchNorm2d(out_features))

    def forward(self, x):
        fea = self.feas(x)
        return fea + self.shortcut(x)

class ResNeXt(nn.Module):

    def __init__(self, class_num):
        super(ResNeXt, self).__init__()
        self.basic_conv = nn.Sequential(nn.Conv2d(3, 64, 3, padding=1), nn.BatchNorm2d(64))
        self.stage_1 = nn.Sequential(ResNeXtUnit(64, 256, mid_features=128), nn.ReLU(), ResNeXtUnit(256, 256), nn.ReLU(), ResNeXtUnit(256, 256), nn.ReLU())
        self.stage_2 = nn.Sequential(ResNeXtUnit(256, 512, stride=2), nn.ReLU(), ResNeXtUnit(512, 512), nn.ReLU(), ResNeXtUnit(512, 512), nn.ReLU())
        self.stage_3 = nn.Sequential(ResNeXtUnit(512, 1024, stride=2), nn.ReLU(), ResNeXtUnit(1024, 1024), nn.ReLU(), ResNeXtUnit(1024, 1024), nn.ReLU())
        self.pool = nn.AvgPool2d(8)
        self.classifier = nn.Sequential(nn.Linear(1024, class_num))

    def forward(self, x):
        fea = self.basic_conv(x)
        fea = self.stage_1(fea)
        fea = self.stage_2(fea)
        fea = self.stage_3(fea)
        fea = self.pool(fea)
        fea = torch.squeeze(fea)
        fea = self.classifier(fea)
        return fea

class SKConv(nn.Module):

    def __init__(self, features, WH, M, G, r, stride=1, L=32):
        """ Constructor
        Args:
            features: input channel dimensionality.
            WH: input spatial dimensionality, used for GAP kernel size.
            M: the number of branchs.
            G: num of convolution groups.
            r: the radio for compute d, the length of z.
            stride: stride, default 1.
            L: the minimum dim of the vector z in paper, default 32.
        """
        super(SKConv, self).__init__()
        d = max(int(features / r), L)
        self.M = M
        self.features = features
        self.convs = nn.ModuleList([])
        for i in range(M):
            self.convs.append(nn.Sequential(nn.Conv2d(features, features, kernel_size=3 + i * 2, stride=stride, padding=1 + i, groups=G), nn.BatchNorm2d(features), nn.ReLU(inplace=False)))
        self.fc = nn.Linear(features, d)
        self.fcs = nn.ModuleList([])
        for i in range(M):
            self.fcs.append(nn.Linear(d, features))
        self.softmax = nn.Softmax(dim=1)

    def forward(self, x):
        for i, conv in enumerate(self.convs):
            fea = conv(x).unsqueeze_(dim=1)
            if i == 0:
                feas = fea
            else:
                feas = torch.cat([feas, fea], dim=1)
        fea_U = torch.sum(feas, dim=1)
        fea_s = fea_U.mean(-1).mean(-1)
        fea_z = self.fc(fea_s)
        for i, fc in enumerate(self.fcs):
            vector = fc(fea_z).unsqueeze_(dim=1)
            if i == 0:
                attention_vectors = vector
            else:
                attention_vectors = torch.cat([attention_vectors, vector], dim=1)
        attention_vectors = self.softmax(attention_vectors)
        attention_vectors = attention_vectors.unsqueeze(-1).unsqueeze(-1)
        fea_v = (feas * attention_vectors).sum(dim=1)
        return fea_v

class SKUnit(nn.Module):

    def __init__(self, in_features, out_features, WH, M, G, r, mid_features=None, stride=1, L=32):
        """ Constructor
        Args:
            in_features: input channel dimensionality.
            out_features: output channel dimensionality.
            WH: input spatial dimensionality, used for GAP kernel size.
            M: the number of branchs.
            G: num of convolution groups.
            r: the radio for compute d, the length of z.
            mid_features: the channle dim of the middle conv with stride not 1, default out_features/2.
            stride: stride.
            L: the minimum dim of the vector z in paper.
        """
        super(SKUnit, self).__init__()
        if mid_features is None:
            mid_features = int(out_features / 2)
        self.feas = nn.Sequential(nn.Conv2d(in_features, mid_features, 1, stride=1), nn.BatchNorm2d(mid_features), SKConv(mid_features, WH, M, G, r, stride=stride, L=L), nn.BatchNorm2d(mid_features), nn.Conv2d(mid_features, out_features, 1, stride=1), nn.BatchNorm2d(out_features))
        if in_features == out_features:
            self.shortcut = nn.Sequential()
        else:
            self.shortcut = nn.Sequential(nn.Conv2d(in_features, out_features, 1, stride=stride), nn.BatchNorm2d(out_features))

    def forward(self, x):
        fea = self.feas(x)
        return fea + self.shortcut(x)

class SKNet(nn.Module):

    def __init__(self, class_num):
        super(SKNet, self).__init__()
        self.basic_conv = nn.Sequential(nn.Conv2d(3, 64, 3, padding=1), nn.BatchNorm2d(64))
        self.stage_1 = nn.Sequential(SKUnit(64, 256, 32, 2, 8, 2, stride=2), nn.ReLU(), SKUnit(256, 256, 32, 2, 8, 2), nn.ReLU(), SKUnit(256, 256, 32, 2, 8, 2), nn.ReLU())
        self.stage_2 = nn.Sequential(SKUnit(256, 512, 32, 2, 8, 2, stride=2), nn.ReLU(), SKUnit(512, 512, 32, 2, 8, 2), nn.ReLU(), SKUnit(512, 512, 32, 2, 8, 2), nn.ReLU())
        self.stage_3 = nn.Sequential(SKUnit(512, 1024, 32, 2, 8, 2, stride=2), nn.ReLU(), SKUnit(1024, 1024, 32, 2, 8, 2), nn.ReLU(), SKUnit(1024, 1024, 32, 2, 8, 2), nn.ReLU())
        self.pool = nn.AvgPool2d(8)
        self.classifier = nn.Sequential(nn.Linear(1024, class_num))

    def forward(self, x):
        fea = self.basic_conv(x)
        fea = self.stage_1(fea)
        fea = self.stage_2(fea)
        fea = self.stage_3(fea)
        fea = self.pool(fea)
        fea = torch.squeeze(fea)
        fea = self.classifier(fea)
        return fea

import torch
from torch.nn import MSELoss, ReLU
from _paritybench_helpers import _mock_config, _mock_layer, _paritybench_base, _fails_compile

TESTCASES = [
    # (nn.Module, init_args, forward_args, jit_compiles)
    (ResNeXtUnit,
     lambda: ([], {'in_features': 64, 'out_features': 64}),
     lambda: ([torch.rand([4, 64, 64, 64])], {}),
     True),
    (SKConv,
     lambda: ([], {'features': 4, 'WH': 4, 'M': 4, 'G': 4, 'r': 4}),
     lambda: ([torch.rand([4, 4, 4, 4])], {}),
     False),
    (SKNet,
     lambda: ([], {'class_num': 4}),
     lambda: ([torch.rand([4, 3, 64, 64])], {}),
     False),
]

class Test_pppLang_SKNet(_paritybench_base):
    def test_000(self):
        self._check(*TESTCASES[0])

    def test_001(self):
        self._check(*TESTCASES[1])

    def test_002(self):
        self._check(*TESTCASES[2])


import os
import plotly as py
import plotly.io as pio
import plotly.graph_objects as go
pyplot = py.offline.plot

if os.path.exists("../images"):
    i_path="../images"
else:
    i_path="."
if os.path.exists("../htmls"):
    h_path = "../htmls"
else:
    h_path = "."

#处理数据
names=['Alex','Ok Google','Choice','Describe','Guess','Hey Siri']
file_name='arouse.csv'
if os.path.exists("../data"):
    with open("../data/"+file_name) as f:
        text=f.readlines()
else:
    with open(file_name) as f:
        text=f.readlines()
data=[]
for line in text:
    t=[]
    line=line.split(',')
    for d in line[:-2]:
        if d=='':
            pass
        else:
            t.append(float(d)*100)
    data.append(t)
colors=['rgba(247,183,112,1)','rgba(187,189,191,1)','rgba(233,155,122,1)',
        'rgba(236,111,70,1)','rgba(178,170,107,1)','rgba(143,238,146,1)','rgba(93,156,204,1)']
#画图
fig = go.Figure()
for d,name,color in zip(data,names,colors):
    fig.add_trace(go.Box(y=d,
                         name=name,
                         fillcolor=color,
                         # line_color=color,
                         boxpoints='all'))

#设置参数
fig.update_layout(
                showlegend=False,
                height=630 ,width = 750,    #画布大小
                font=dict(
                    family="Times New Roman",  # 所有标题文字的字体
                    size = 32, # 所有标题文字的大小
                ),
                template='simple_white',
                yaxis=dict(
                    range=[90,100]
                )
                )
fig.update_xaxes(showgrid=True,#将网格去掉
                 linewidth=1.5,
                 linecolor='black', # 将颜色设定为黑色
                 mirror=True,
                 gridcolor='#dbddde',
                 )     # 加上这个  四周都是黑色  ，不加的话只有左下两条线黑色  （就是镜像过去）
fig.update_yaxes(title='Accuracy(%)',
                 showgrid=True,
                 linewidth=1.5,
                 linecolor='black',
                 mirror=True,
                 gridcolor='#dbddde',
                 )
html_path = os.path.join(h_path,"ImpactofArouse.html")
pio.write_image(fig,os.path.join(i_path,'ImpactofArouse.eps'))
pyplot(fig,filename=html_path)


from rest_framework import serializers

from apps.alerts.models import AlertGroup, ResolutionNote
from apps.api.serializers.user import FastUserSerializer
from apps.mobile_app.auth import MobileAppAuthTokenAuthentication
from common.api_helpers.custom_fields import OrganizationFilteredPrimaryKeyRelatedField
from common.api_helpers.exceptions import BadRequest
from common.api_helpers.mixins import EagerLoadingMixin

class ResolutionNoteSerializer(EagerLoadingMixin, serializers.ModelSerializer):
    id = serializers.CharField(read_only=True, source="public_primary_key")
    alert_group = OrganizationFilteredPrimaryKeyRelatedField(
        filter_field="channel__organization",
        queryset=AlertGroup.objects,
    )
    text = serializers.CharField(allow_null=False, source="message_text")
    author = FastUserSerializer(read_only=True)

    SELECT_RELATED = ["resolution_note_slack_message", "author"]

    class Meta:
        model = ResolutionNote
        fields = [
            "id",
            "alert_group",
            "source",
            "author",
            "created_at",
            "text",
        ]
        read_only_fields = [
            "author",
            "created_at",
            "source",
        ]

    def create(self, validated_data):
        validated_data["author"] = self.context["request"].user

        if isinstance(self.context["request"].successful_authenticator, MobileAppAuthTokenAuthentication):
            source = ResolutionNote.Source.MOBILE_APP
        else:
            source = ResolutionNote.Source.WEB
        validated_data["source"] = source

        created_instance = super().create(validated_data)
        return created_instance

    def to_representation(self, instance):
        result = super().to_representation(instance)
        result["text"] = instance.text
        result["source"] = {"id": instance.source, "display_name": instance.get_source_display()}
        return result

class ResolutionNoteUpdateSerializer(ResolutionNoteSerializer):
    alert_group = OrganizationFilteredPrimaryKeyRelatedField(read_only=True)

    def update(self, instance, validated_data):
        if instance.source != ResolutionNote.Source.WEB:
            raise BadRequest(detail="Cannot update message with this source type")

        return super().update(instance, validated_data)