from django.shortcuts import get_object_or_404
from django.core.mail import send_mail
from rest_framework import generics, status
from rest_framework.permissions import IsAdminUser, AllowAny, BasePermission
from rest_framework.response import Response
from rest_framework.views import APIView
from .serializers import (
    FoodSerializer, TagSerializer, OptionalItemsSerializer, OrderedFoodSerializer,
    FoodCartSerializer, DeliveryEntitySerializer, OngoingOrderSerializer,
    CustomerSerializer, DeliveryPersonSerializer, LoginSerializer
)
from Orders.models import (
    Food, Tag, OptionalItem, OrderedFood, FoodCart, OrderedOptionalItem, OngoingOrder,
    DeliveryEntity
)
from rest_framework.permissions import IsAuthenticated
from Orders.documents import FoodDocument

# Create your views here.

class IsOwnerOrIPMixin(BasePermission):
    def has_object_permission(self, request, view, obj):
        if request.user.is_authenticated:
            return obj.owner == request.user.customer
        else:
            ip_address = request.META.get('REMOTE_ADDR')
            return obj.ip_address == ip_address

class FoodListAPIVIew(generics.ListAPIView):
    queryset = Food.active_objects.all()
    serializer_class = FoodSerializer

class DetailAPIView(generics.RetrieveAPIView):
    serializer_class = FoodSerializer
    queryset = Food.active_objects.all()
    permission_classes = [AllowAny]

    def get(self, request, *args, **kwargs):
        return self.retrieve(request, *args, **kwargs)

    def post(self, request, pk):
        food = Food.objects.get(id=pk)
        food_quantity = int(request.data.get('quantity'))
        optional_items = []
        optional_item_quantities = []
        ip_address = request.META.get('REMOTE_ADDR')
        for key, value in request.data.items():
            if key.startswith('optional_item_'):
                optional_item_id = key.split('_')[-1]
                optional_item = OptionalItem.objects.get(id=optional_item_id)
                optional_items.append(optional_item)
                optional_item_quantities.append(int(value))

        ordered_food_kwargs = {
            'food': food,
            'food_quantity': food_quantity,
        }
        if request.user.is_authenticated:
            ordered_food_kwargs['user'] = request.user.customer
        else:
            ordered_food_kwargs['ip_address'] = ip_address

        ordered_food = OrderedFood.objects.create(**ordered_food_kwargs)

        for item, quantity in zip(optional_items, optional_item_quantities):
            ordered_optional_item = OrderedOptionalItem.objects.create(
                ordered_food=ordered_food,
                optional_item=item,
                quantity=quantity
            )

        if request.user.is_authenticated:
            selected_cart = FoodCart.objects.filter(user=request.user.customer, is_checked_out=False).first()
            if selected_cart is not None:
                existing_ordered_food = selected_cart.ordered_food.filter(food=ordered_food.food).first()
                if existing_ordered_food is not None:
                    existing_ordered_food.food_quantity += ordered_food.food_quantity
                    existing_ordered_optional_items = existing_ordered_food.get_ordered_optional_items()

                    for ordered_optional_item in ordered_food.get_ordered_optional_items():
                        existing_optional_item = existing_ordered_optional_items.filter(
                            optional_item=ordered_optional_item.optional_item).first()
                        if existing_optional_item is not None:
                            existing_optional_item.quantity += ordered_optional_item.quantity
                            existing_optional_item.save()
                        else:
                            existing_ordered_food.optional_items.add(ordered_optional_item.optional_item,
                                                                     through_defaults={
                                                                         'quantity': ordered_optional_item.quantity})
                    existing_ordered_food.save()
                    return Response({'message': f'{ordered_food} successfully added to your cart'},
                                    status=status.HTTP_201_CREATED)

                selected_cart.ordered_food.add(ordered_food)
                return Response({'message': f'{ordered_food} successfully added'}, status=status.HTTP_201_CREATED)

            new_cart = FoodCart.objects.create(user=request.user.customer)
            new_cart.ordered_food.add(ordered_food)
            new_cart.save()
            return Response({'message': f'{ordered_food} successfully added to your cart'},
                            status=status.HTTP_201_CREATED)
        else:

            selected_cart = FoodCart.objects.filter(ip_address=ip_address,
                                                    is_checked_out=False).first()
            if selected_cart is not None:
                existing_ordered_food = selected_cart.ordered_food.filter(food=ordered_food.food).first()
                print('yes', existing_ordered_food)
                if existing_ordered_food is not None:
                    existing_ordered_food.food_quantity += ordered_food.food_quantity
                    existing_ordered_optional_items = existing_ordered_food.get_ordered_optional_items()
                    for ordered_optional_item in ordered_food.get_ordered_optional_items():
                        existing_optional_item = existing_ordered_optional_items.filter(
                            optional_item=ordered_optional_item.optional_item).first()
                        if existing_optional_item is not None:
                            existing_optional_item.quantity += ordered_optional_item.quantity
                            existing_optional_item.save()
                        else:
                            existing_ordered_food.optional_items.add(ordered_optional_item.optional_item,
                                                                     through_defaults={
                                                                         'quantity': ordered_optional_item.quantity})
                    existing_ordered_food.save()
                    return Response({'message': f'{ordered_food} successfully added to your cart'},
                                    status=status.HTTP_201_CREATED)
                else:
                    selected_cart.ordered_food.add(ordered_food)
                    return Response({'message': f'{ordered_food} successfully added to your cart'},
                                    status=status.HTTP_201_CREATED)
            else:
                new_cart = FoodCart.objects.create(ip_address=ip_address)
                new_cart.ordered_food.add(ordered_food)
                new_cart.save()
                return Response({'message': f'{ordered_food} successfully added to your cart'},
                                status=status.HTTP_201_CREATED)

class FoodListCreateUpdateDeleteAPIView(generics.ListAPIView, generics.CreateAPIView, generics.UpdateAPIView,
                                        generics.DestroyAPIView):
    queryset = Food.objects.all()
    serializer_class = FoodSerializer
    permission_classes = [IsAdminUser]

class TagListAPIview(generics.ListAPIView):
    queryset = Tag.objects.all()
    serializer_class = TagSerializer
    permission_classes = [IsAdminUser]

class TagDetailUpdateDeleteAPIView(generics.RetrieveUpdateDestroyAPIView):
    queryset = Tag.objects.all()
    serializer_class = TagSerializer
    permission_classes = [IsAdminUser]

class OptionalItemListAPIView(generics.ListAPIView):
    queryset = OptionalItem.objects.all()
    serializer_class = OptionalItemsSerializer
    permission_classes = [IsAdminUser]

class OptionalItemDetailUpdateDeleteAPIView(generics.RetrieveUpdateDestroyAPIView):
    queryset = OptionalItem.objects.all()
    serializer_class = OptionalItemsSerializer
    permission_classes = [IsAdminUser]

class OrderedFoodCreateAPIView(generics.CreateAPIView):
    queryset = OrderedFood.objects.all()
    serializer_class = OrderedFoodSerializer

    def perform_create(self, serializer):
        if self.request.user.is_authenticated:
            serializer.save(user=self.request.user)
        else:
            serializer.save(ip_address=self.request.META('REMOTE ADDRS', None))

class FoodCartAPIView(IsOwnerOrIPMixin, generics.RetrieveAPIView):
    queryset = FoodCart.objects.all()
    serializer_class = FoodCartSerializer

    def get_object(self):
        if self.request.user.is_authenticated:
            user = self.request.user.customer
            cart = FoodCart.objects.filter(user=user, is_checked_out=False).first()
            print('user is authenticated')
        else:
            ip_address = self.request.META.get('REMOTE_ADDR')
            cart = FoodCart.objects.filter(ip_address=ip_address, is_checked_out=False).first()
            print('user is not authenticated')

        if cart is None:
            return Response({'message': "Cart not found"}, status=status.HTTP_404_NOT_FOUND)
        return cart

class RemoveFoodCartItemAPIView(IsOwnerOrIPMixin, generics.DestroyAPIView):
    queryset = OrderedFood.objects.all()
    serializer_class = OrderedFoodSerializer

    def delete(self, request, ordered_food_id):
        ordered_food = get_object_or_404(OrderedFood, id=ordered_food_id)
        ordered_food.delete()
        return Response({'message': f'{ordered_food} has been removed'}, status=status.HTTP_200_OK)

class PlaceFoodOrderAPIView(IsOwnerOrIPMixin, generics.CreateAPIView):
    queryset = DeliveryEntity.objects.all()
    serializer_class = DeliveryEntitySerializer

    def perform_create(self, serializer):
        food_cart = get_object_or_404(FoodCart, id=self.kwargs.get('food_cart_id'))
        if self.request.user.is_authenticated:
            owner = self.request.user.customer
            phone_number = self.request.user.phone_number
            address = self.request.user.address
            ip_address = None
        else:
            owner = None
            ip_address = self.request.META.get('REMOTE_ADDR')
        data = {'food_cart': food_cart, 'owner': owner, 'address': address, 'phone_number': phone_number,
                'ip_address': ip_address, 'is_active': True}
        food_cart.is_checked_out = True
        food_cart.save()
        serializer.save(**data)
        return Response({'message': "Order has been placed successfully"}, status=status.HTTP_201_CREATED)

class UserDashboardAPIView(IsOwnerOrIPMixin, generics.RetrieveAPIView):
    serializer_class = DeliveryEntitySerializer

    def get_object(self):
        if self.request.user.is_authenticated:
            return DeliveryEntity.objects.filter(owner=self.request.user.customer, is_active=True).first()
        else:
            return DeliveryEntity.objects.filter(ip_address=self.request.META.get('REMOTE_ADDR'),
                                                 is_active=True).first()

    def retrieve(self, request, *args, **kwargs):
        instance = self.get_object()
        serializer = self.get_serializer(instance)
        return Response(serializer.data)

class UserAcceptDeliveryAPIView(IsOwnerOrIPMixin, generics.UpdateAPIView):
    queryset = DeliveryEntity.objects.all()
    serializer_class = DeliveryEntitySerializer
    lookup_field = 'pk'

    def update(self, request, *args, **kwargs):
        delivery_entity = self.get_object()
        if delivery_entity.is_accepted and delivery_entity.ongoing_status:
            delivery_entity.accept_delivery = True
            delivery_entity.is_active = False
            delivery_entity.save()
            response_data = {'message': ' You have accepted the delivery'}
            return Response(response_data, status=status.HTTP_200_OK)
        else:
            response_data = {'message': 'You can not accept delivery at this time'}
            return Response(response_data, status=status.HTTP_404_NOT_FOUND)

class AdminAcceptOrderAPIView(generics.UpdateAPIView):
    queryset = DeliveryEntity
    serializer_class = DeliveryEntitySerializer
    lookup_field = 'pk'
    permission_classes = [IsAdminUser]

    def update(self, request, *args, **kwargs):
        delivery_entity = self.get_object()
        delivery_entity.is_accepted = True
        delivery_entity.ongoing_status = True
        delivery_entity.save()
        response_data = {'message': 'You have accepted this order, delivery status is: ongoing'}
        return Response(response_data, status=status.HTTP_200_OK)

class DeliveryEntityDetailAPIView(generics.RetrieveAPIView):
    queryset = DeliveryEntity.objects.all()
    serializer_class = DeliveryEntitySerializer

class OngoingOrderListAPIView(generics.ListAPIView):
    queryset = OngoingOrder.objects.all()
    serializer_class = OngoingOrderSerializer
    permission_classes = [IsAdminUser]

class OngoingOrderDetailAPIView(generics.RetrieveAPIView):
    queryset = OngoingOrder.objects.all()
    serializer_class = OngoingOrderSerializer
    permission_classes = [IsAdminUser]

class CancelOngoingOrderAPIView(generics.UpdateAPIView):
    queryset = OngoingOrder.objects.all()
    serializer_class = OngoingOrderSerializer
    permission_classes = [IsAdminUser]

    def update(self, request, *args, **kwargs):
        ongoing_order = self.get_object()
        cancel_reason = request.data.get('cancel_reason', '')

        if cancel_reason:
            ongoing_order.cancel(cancel_reason)
            subject = 'Your ongoing Food order With tao Kitchen has been cancelled'
            message = f'Your ongoing Food order with ID {ongoing_order.id} has been cancelled for {cancel_reason}'
            from_email = 'otutaiwo1@gmail.com'
            recipient_list = [ongoing_order.delivery_entity.owner.user.email]
            send_mail(subject, message, from_email, recipient_list)

            ongoing_order.delivery_entity.ongoing_status = False
            ongoing_order.delivery_entity.save()

            serializer = self.get_serializer(ongoing_order)
            return Response(serializer.data, status=status.HTTP_200_OK)
        else:
            return Response({'error': 'Please select a cancel reason.'}, status=status.HTTP_404_NOT_FOUND)

class CustomerRegistrationAPIView(APIView):
    def post(self, request):
        serializer = CustomerSerializer(data=request.data)
        if serializer.is_valid():
            customer = serializer.save()
            response_data = {
                'message': 'Customer account created successfully',
                'customer_id': customer.id
            }
            return Response(response_data, status=status.HTTP_201_CREATED)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class DeliveryPersonCreateAccountAPIView(APIView):
    def post(self, request):
        serializer = DeliveryPersonSerializer(data=request.data)
        if serializer.is_valid():
            delivery_person = serializer.save()
            return Response(serializer.data, status=status.HTTP_201_CREATED)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class CustomerLoginAPIView(generics.CreateAPIView):
    serializer_class = LoginSerializer

    def create(self, request, *args, **kwargs):
        serializer = self.get_serializer(data=request.data)
        if serializer.is_valid():
            return Response(serializer.validated_data, status=status.HTTP_200_OK)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class CustomerDetailsUpdateAPIView(generics.UpdateAPIView):
    serializer_class = CustomerSerializer
    permission_classes = [IsAuthenticated]

    def get_object(self):
        return self.request.user.customer

    def perform_update(self, serializer):
        serializer.save()

class FoodSearchAPIView(APIView):
    def get(self, request):
        search_query = request.GET.get('search_query')
        if search_query:
            search = FoodDocument.search().query("match", name=search_query)
            results = search.execute()
            food_ids = [hit.meta.id for hit in results]
            foods = Food.active_objects.filter(id__in=food_ids)
            if foods:
                serialized_foods = FoodSerializer(foods, many=True, context={'request': request})
                return Response(serialized_foods.data)
            else:
                response_data = {'message': 'No food with this name is found'}
                return Response(response_data, status=status.HTTP_404_NOT_FOUND)
        response_data = {'message': 'Please provide a search query'}
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)


#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@File    :   object_oriented_basic_demo02.py
@Contact :   wangdamar@gmail.com
@License :   GPL v3

@Modify Time      @Author    @Version    @Desciption
------------      -------    --------    -----------
2020/10/11 9:53   banxian      1.0         None
'''

'''
定义一个天山童姥类 ，类名为TongLao，属性有血量，武力值（通过传入的参数得到）。TongLao类里面有2个方法，
see_people方法，需要传入一个name参数，如果传入”WYZ”（无崖子），则打印，“师弟！！！！”，如果传入“李秋水”，打印“呸，贱人”，如果传入“丁春秋”，打印“叛徒！我杀了你”
fight_zms方法（天山折梅手），调用天山折梅手方法会将自己的武力值提升10倍，血量缩减2倍。需要传入敌人的hp，power，进行一回合制对打，打完之后，比较双方血量。血多的一方获胜。

定义一个XuZhu类，继承于童姥。虚竹宅心仁厚不想打架。所以虚竹只有一个read（念经）的方法。每次调用都会打印“罪过罪过”
加入模块化改造
'''

class TongLao(object):
    def __init__(self,hp,power):
        self.hp = hp
        self.power = power
    def see_people(self,name):
        if(name == "WYZ"):
            print("师弟！！！！")
        if(name == "LQS"):
            print("呸，贱人")
        if(name == "DCQ"):
            print("叛徒！我杀了你")
    def fight_zms(self,enemy_hp,enemy_power):
        self.hp /= 2
        self.power *= 2
        self.hp -= enemy_power
        enemy_hp -= self.power
        if(self.hp > enemy_hp):
            print("I win")
        elif(self.hp < enemy_hp):
            print("I lose")
        else:
            print("nobody win")

class XuZhu(TongLao):
    def read(self):
        print("罪过罪过")

if __name__ == '__main__':
    tl = TongLao(200,5000)
    xz = XuZhu(3000,10)
    tl.see_people("LQS")
    xz.read()


import os
import torch
from torch.utils.data import Dataset
from torchvision.transforms import transforms
import numpy as np
import collections
from PIL import Image
import csv
import random

dataNumpies = {}

class MiniImagenetNP(Dataset):
    """
    put mini-imagenet files as :
    root :
        |- train.npy
        |- test.npy
        |- val.npy
    NOTICE: meta-learning is different from general supervised learning, especially the concept of batch and set.
    batch: contains several sets
    sets: conains n_way * k_shot for meta-train set, n_way * n_query for meta-test set.
    """

    def __init__(self, root, mode, transform):
        """

        :param root: root path of mini-imagenet
        :param mode: train, val or test
        :param resize: resize to
        """
        global dataNumpies
        self.mode = mode
        self.transform = transform
        print (root, mode)
        if mode not in dataNumpies:
            path = os.path.join(root, mode + ".npy")  # image path
            dictionary = np.load(path, allow_pickle=True).item()
            dataNumpies[mode] = list(dictionary.values())

        self.elements_per_class = len(dataNumpies[mode][0])
        self.active_classes = list(range(len(dataNumpies[mode])))

        print("Total classes = ", len(self.active_classes))

    def __getitem__(self, index):
        """
        index means index of sets, 0<= index <= batchsz-1
        :param index:
        :return:

        """
        outer_index = int(index/self.elements_per_class)
        inner_index = index%self.elements_per_class

        label = self.active_classes[outer_index]
        image = dataNumpies[self.mode][label][inner_index]
        image = self.transform(image)
        return image, label

    def __len__(self):
      return len(self.active_classes) * self.elements_per_class

if __name__ == '__main__':
  mini = MiniImagenet('/content/MiniImageNet_dataset/', mode='val', resize=84)
  data = next(iter(mini))


import logging
import re
import sys
from io import StringIO
from types import ModuleType
from typing import Callable, List, TextIO, Tuple

from ..io_utils import get_lines

logger = logging.getLogger(__name__)
VAR = r"(w|x|y|z)"
INT = r"(\-?\d+)"
VAR_OR_INT = rf"({VAR}|{INT})"

INSTRUCTIONS: List[Tuple[str, str]] = [
    (rf"^inp (?P<a>{VAR})$", "{a} = inputs.pop(0)"),
    (rf"^add (?P<a>{VAR}) (?P<b>{VAR_OR_INT})$", "{a} += {b}"),
    (rf"^mul (?P<a>{VAR}) (?P<b>{VAR_OR_INT})$", "{a} *= {b}"),
    (rf"^div (?P<a>{VAR}) (?P<b>{VAR_OR_INT})$", "{a} //= {b}"),
    (rf"^mod (?P<a>{VAR}) (?P<b>{VAR_OR_INT})$", "{a} %= {b}"),
    (rf"^eql (?P<a>{VAR}) (?P<b>{VAR_OR_INT})$", "{a} = int({a} == {b})"),
]

def parse_line(line: str) -> str:
    for pattern, format_string in INSTRUCTIONS:
        match = re.match(pattern, line)
        if match is not None:
            break
    else:
        raise AssertionError(f"Line not matched {line!r}")
    return format_string.format(**match.groupdict())

def parse_program(buf: StringIO, input: TextIO) -> None:
    for line in get_lines(input):
        code = parse_line(line)
        buf.write(f"    {code}\n")

def convert_to_python(input: TextIO) -> str:
    buf = StringIO()
    buf.write("from typing import List, Tuple\n")
    buf.write("\n")
    buf.write("def program(inputs: List[int]) -> Tuple[int, int, int, int]:\n")
    buf.write("    w = x = y = z = 0\n")
    parse_program(buf, input)
    buf.write("    return w, x, y, z\n")
    return buf.getvalue()

def compile_program(input: TextIO) -> Callable[[List[int]], Tuple[int, int, int, int]]:
    module_name = "fake"
    python_text = convert_to_python(input)
    logger.debug("Python code:\n%s", python_text)
    code = compile(python_text, "<string>", "exec")
    module = ModuleType(module_name)
    exec(code, module.__dict__)
    sys.modules[module_name] = module
    program: Callable[[List[int]], Tuple[int, int, int, int]] = getattr(
        module, "program"
    )
    return program


class Stack:
    def __init__(self):
        self.stack = []

    def push(self, data):
        self.stack.append(data)

    def pop(self):
        if self.stack:
            return self.stack.pop()

"""
Input {'key1': 'value1', 'key2': [1, 2, 3], 'key3': (1, 2, 3)} -> True
Input {'key1': ['value1', 'key2': [1, 2, 3], 'key3': (1, 2, 3)} -> False
"""

def validate_format(chars: str) -> bool:
    lookup = {'{': '}', '(': ')', '[': ']'}
    stack = []
    for char in chars:
        if char in lookup.keys():
            stack.append(lookup[char])
        elif char in lookup.values():
            if not stack:
                return False
            if char != stack.pop():
                return False

    if stack:
        return False

    return True

class Queue:
    def __init__(self):
        self.queue = []

    def enqueue(self, data):
        self.queue.append(data)

    def dequeue(self):
        """
        先頭から取り出す
        """
        self.queue.pop(0)

    def reverse(self):
        new_queue = []
        while self.queue:
            new_queue.append(self.queue.pop())
        self.queue = new_queue

q = Queue()
q.enqueue(1)
q.enqueue(2)
q.enqueue(3)
q.reverse()
q.dequeue()


"""Mediciones para el tiempo de Secado en horas de una marca de pintura esmaltada
    a)Tamaño de la muestra
    b)Media
    c)Mediana
    d)Grafica de los datos
    e)Media Recortada al 20%
    f)Que representa mejor los datos Media muestral o Media Recortada """

from location_measures import average, mediana, average_recort
from bokeh.plotting import figure, output_file, show

def grafic(measure):
    output_file = ('variance-standard_deviation-Exercise1.1.html')
    fig = figure()

    x_vals = list(measure)
    y_vals = 0
    fig.dot(x_vals, y_vals, size = 20)
    show(fig)

def main(measure):
    return len(measure)

if __name__ == "__main__":
    measure = [3.4, 2.5, 4.8, 2.9, 3.6, 2.8, 3.3, 5.6, 3.7, 2.8, 4.4, 4.0, 5.2, 3.0, 4.8]

    print(f'The measuraments are: {measure}')
    #a)
    print(f'The size of the sample is: {main(measure)}')
    #b)
    print(f'The average is: {average(measure)}')
    #c
    print(f'The median is: {mediana(measure)}')
    #d
    grafic(measure)
    #e
    print(f"The media recort is: {average_recort(measure)}")


from datetime import timedelta as td, datetime

from django.utils import timezone
from django.db.models.expressions import F

from config.settings import LAST_ACTIVITY_INTERVAL
from users.models import Profile
from users.constants import MAXIMUM_DAY

def last_user_activity_middleware(get_response):

    def middleware(request):

        response = get_response(request)

        expected_entry_date = datetime.today().date() - td(seconds=LAST_ACTIVITY_INTERVAL)

        if request.user.is_authenticated and request.user.profile.last_login == expected_entry_date:
            Profile.objects.filter(pk=request.user.pk).update(last_login=timezone.now(), login_in_row=F('login_in_row') + 1)
            request.user.refresh_from_db()
            if request.user.profile.login_in_row == MAXIMUM_DAY:
                request.user.profile.login_in_row = 0
                request.user.refresh_from_db()
        elif request.user.is_authenticated and request.user.profile.last_login < expected_entry_date:
            request.user.profile.login_in_row = 0
            request.user.refresh_from_db()

        return response

    return middleware


import operator

class AVLNode():
    def __init__(self, val, key=operator.lt, left=None, right=None):
        self.val = val
        self.key = key
        # enforce that the key is an invariant of the tree
        for subroot in [x for x in (left, right) if x]:
            assert subroot.key == key

        # now assert the key behaves as expected
        if left:
            assert not key(left.val, val)
        if right:
            assert key(right.val, val)

        self.left, self.right = left, right

        self.left = left
        self.right = right

        # this is bothersome. is there a better way of doing this?
        if left:
            if right:
                self.height = 1 + max(left.height, right.height)
            else:
                self.height = 1 + left.height
        elif right:
            self.height = 1 + right.height
        else:
            self.height = 0

        # an assumption I'm making here is that we don't need to rebalance
        # our left and right subtrees
        # but in case that assumption is scary:
        # self.maybe_rebalance()

    def insert(self, newval):
        if self.key(newval, self.val):
            if self.right:
                self.right.insert(newval)
                self.height = max(
                    self.height,
                    self.right.height + 1)
            else:
                # propagate the key
                self.right = AVLNode(newval, key=self.key)
                # this preserves the height unless self was previously a leaf
                self.height = max(self.height, 1)
        else:
            # this can be bumped up above
            # while preserving correctness.
            # I'm going to leave it as is.
            if self.left:
                self.left.insert(newval)
                self.height = max(
                    self.height,
                    self.left.height + 1)
            else:
                self.left = AVLNode(newval, key=self.key)
                self.height = max(self.height, 1)
        self.maybe_rebalance()

    def get_balance(self):
        # if a subtree is none we don't attempt
        # to get the height, which avoids attr error
        # this is... incorrect.
        # if self.left is None the left subtree has a height
        # of -1
        if self.left:
            cands.append(self.left.height)
        return (self.left and self.left.height or 0 -
                self.right and self.right.height or 0)

    def maybe_rebalance(self):
        balance = self.get_balance()
        if balance == -2:
            # the right tree is heavier
            # further we are guaranteed that there is an AVLNode
            # on the right. (so we can right balance)
            self.right_balance()
        elif balance == 2:
            self.left_balance()
        elif balance not in range(-1, 2):
            # can we fix this?
            raise ValueError("This node is broken: ", self)

    # neither of these are right
    # take a break, come back, and address the situations
    # of when and how to rotate a subtree.
    def left_balance(self):
        if self.left and self.left.get_balance() > -1:
            self.left.right_rotate()
        self.right_rotate()

    def right_balance(self):
        if self.right and self.right.get_balance() < 1:
            self.right.left_rotate()
        self.right_rotate()

    # these are correct.
    def left_rotate(self):
        self.left = AVLNode(
            self.val,
            left=self.left,
            right=self.right.left)
        self.val = self.right.val
        self.right = self.right.right

    def right_rotate(self):
        self.right = AVLNode(
            self.val,
            left=self.left.right,
            right=self.right)
        self.val = self.left.val
        self.left = self.left.left


""":mod:`seoul.client` --- Client
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

"""
from datetime import datetime
from typing import List

from requests import Session
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry

from .subway.train import Direction, Train
from .air import CityAirQuality

SAMPLE_API_KEY = 'sample'

SUBWAY_BASE_URL = 'http://swopenapi.seoul.go.kr'
SUBWAY_REALTIME_POSITION_URL = '/api/subway/{api_key}/{format}/realtimePosition/{start_index}/{end_index}/{subway_name}'

AIR_BASE_URL = 'http://openapi.seoul.go.kr:8088'
AIR_REALTIME_CITY_URL = '/{api_key}/{format}/RealtimeCityAir/{start_index}/{end_index}'

class Client:
    def __init__(self, api_key: str = SAMPLE_API_KEY):
        self.api_key = api_key

    def get_air_realtime_city(self, start_index: int = 1, end_index: int = 1000) -> List:
        """Get realtime city air information.

        Specification: http://data.seoul.go.kr/dataList/OA-2219/S/1/datasetView.do

        """
        if self.api_key == SAMPLE_API_KEY:
            start_index = 0
            end_index = 5

        url = AIR_BASE_URL + AIR_REALTIME_CITY_URL.format(
            api_key=self.api_key,
            format='json',
            start_index=start_index,
            end_index=end_index,
        )
        s = Session()
        retries = Retry(status_forcelist=[503])
        s.mount(AIR_BASE_URL, HTTPAdapter(max_retries=retries))
        r = s.get(url)

        measurements = []
        for m in r.json()['RealtimeCityAir']['row']:
            data = {
                'measured_at': datetime.strptime(m['MSRDT'], '%Y%m%d%H%M'),
                'region_name': m['MSRRGN_NM'],
                'station_name': m['MSRSTE_NM'],
                'pm10': m['PM10'],
                'pm25': m['PM25'],
                'o3': m['O3'],
                'no2': m['NO2'],
                'co': m['CO'],
                'so2': m['SO2'],
                'index_name': m['IDEX_NM'],
                'index_value': m['IDEX_MVL'],
                'index_pollutant': m['ARPLT_MAIN'],
            }
            measurements.append(CityAirQuality(**data))
        return measurements

    def get_subway_realtime_position(self, subway_name: str,
                                     start_index: int = 0, end_index: int = 1000) -> List[Train]:
        """Get realtime train position for subway line.

        Specification: http://data.seoul.go.kr/dataList/OA-12601/A/1/datasetView.do

        """
        if self.api_key == SAMPLE_API_KEY:
            start_index = 0
            end_index = 5

        s = Session()
        retries = Retry(status_forcelist=[503])
        s.mount(SUBWAY_BASE_URL, HTTPAdapter(max_retries=retries))
        url = SUBWAY_BASE_URL + SUBWAY_REALTIME_POSITION_URL.format(
            api_key=self.api_key,
            format='json',
            subway_name=subway_name,
            start_index=start_index,
            end_index=end_index,
        )
        r = s.get(url)

        d = r.json()
        if 'realtimePositionList' not in d:
            """
            code: 'INFO-200'
            message: '해당하는 데이터가 없습니다.'

            Returned when the subway line has ended operations for the day.
            """
            if d['status'] == 500 and d['code'] == 'INFO-200':
                return []

            raise Exception(f"{d['code']}: {d['message']}")
        else:
            trains = []
            for t in d['realtimePositionList']:
                data = {
                    'subway_id': t['subwayId'],
                    'subway_name': t['subwayNm'],
                    'station_id': t['statnId'],
                    'station_name': t['statnNm'],
                    'terminal_station_id': t['statnTid'],
                    'terminal_station_name': t['statnTnm'],
                    'number': t['trainNo'],
                    'status': t['trainSttus'],
                    'direction': Direction(int(t['updnLine'])),
                    'updated_at': datetime.strptime(t['recptnDt'], '%Y-%m-%d %H:%M:%S'),
                    'express': t['directAt'] == '1',
                    'last': t['lstcarAt'] == '1',
                }
                trains.append(Train(**data))
            return trains


class App:
    def __init__(self, app_json: dict[str, str] | dict[str, dict[str, int]] = None):
        if app_json:
            self.fromJSON(app_json)
        else:
            self.title = ""
            self.path = ""
            self.hours = 0
            self.minutes = 0

        self.current_hours = 0
        self.current_minutes = 0

    def toJSON(self) -> dict[str, str] | dict[str, dict[str, int]]:
        return {
                "title": self.title,
                "path": self.path,
                "total_time": {
                        "hours": self.hours,
                        "minutes": self.minutes
                    }
            }

    def fromJSON(self, app_json: dict[str, dict[str, int]] | dict[str, str]) -> None:
        self.title = app_json['title']
        self.path = app_json['path']
        self.hours = app_json['total_time']['hours']
        self.minutes = app_json['total_time']['minutes']


import sys
import inspect
import argparse
import pickle
from pathlib import Path
from importlib import import_module
from tqdm import tqdm

import utils
import pipeline
from pipeline import create_image_paths_file, image_paths_from_folders, create_data_path_index, load_index_paths, load_channel_names, save_channel_names
from pipeline import segmentator_setup, get_masks, normalize_images, clean_and_save_masks, crop_images, resize, filter_images_by_sharpness, filter_masks_by_sharpness
from stats import pixel_range_info, normalization_dry_run, image_by_level_set, sample_sharpness, sharpness_dry_run
import seaborn as sns
import matplotlib.pyplot as plt

import torch
import numpy as np
from torch.utils.data import DataLoader

utils.silent = True
pipeline.suppress_warnings = True

stats_opt = ['norm', 'pix_range', 'int_img', 'sample', 'sharp', 'name']
stats_opt_desc = {
    'norm': 'Normalize images and show results and statistics',
    'pix_range': 'Show pixel range statistics, esp percentile intensities per channel',
    'int_img': 'Show image level intensity statistics',
    'sample': 'Show sample images from final dataset',
    'sharp': 'Show sharpness statistics',
    'name': 'Show dataset names'
}
stats_desc = '\n'.join([f"{opt}: {stats_opt_desc[opt]};" for opt in stats_opt])
NORM, PIX_RANGE, INT_IMG, SAMPLE, SHARP, NAME = 0, 1, 2, 3, 4, 5

parser = argparse.ArgumentParser(description='Dataset preprocessing pipline')
parser.add_argument('--data_dir', type=str, help='Path to dataset, should be absolute path', required=True)
parser.add_argument('--output_dir', type=str , help='Path to output directory, should be absolute path')
parser.add_argument('--config', type=str, help='Path to config file, should be absolute path')
parser.add_argument('--name', type=str, help='Name of dataset version to look up in dataset folder (used for cached results)', default='unspecified')
parser.add_argument('--stats', type=str, help=f"Image stats to show, options include: {stats_opt}\n{stats_desc}", choices=stats_opt)
parser.add_argument('--viz_num', type=int, default=5, help='Number of samples to show')
parser.add_argument('--calc_num', type=int, default=30, help='Number of samples to use for calculating image stats')
parser.add_argument('--all', action='store_true', help='Run all steps')
parser.add_argument('--image_mask_cache', action='store_true', help='Save images')
parser.add_argument('--clean_masks', action='store_true', help='Clean masks: remove small objects and join cells without nuclei, etc.')
parser.add_argument('--filter_sharpness', action='store_true', help='Filter out blurry images based on config sharpness threshold')
parser.add_argument('--normalize', action='store_true', help='Normalize images')
parser.add_argument('--single_cell', action='store_true', help='Crop and save single cell images')
parser.add_argument('--rgb', action='store_true', help='Convert images to RGB')
parser.add_argument('--dino_cls', action='store_true', help='Cache dino cls embeddings')
parser.add_argument('--dino_cls_ref', action='store_true', help='Cache dino cls embeddings on reference channels only')
parser.add_argument('--fucci_gmm', action='store_true', help='Fit GMM to FUCCI intensities')
parser.add_argument('--batch_size', type=int, default=10, help='Batch size for dino inference')
parser.add_argument('--device', type=int, default=7, help='GPU device number')
parser.add_argument('--rebuild', action='store_true', help='Rebuild specifed steps even if files exist')
parser.add_argument('--save_samples', action='store_true', help='Save sample outputs for each well')

args = parser.parse_args()

#===================================================================================================
# Basic Setup
#===================================================================================================
DATA_DIR = Path(args.data_dir)
if not DATA_DIR.is_absolute():
    DATA_DIR = Path.cwd() / DATA_DIR
    print(f"Converted relative path to absolute path: {DATA_DIR}")
if not DATA_DIR.exists():
    raise ValueError(f"Data directory {DATA_DIR} does not exist")

OUTPUT_DIR = Path(args.output_dir) if args.output_dir is not None else Path.cwd() / "output"
if not OUTPUT_DIR.exists():
    OUTPUT_DIR.mkdir(parents=True)
    print(f"Created output directory {OUTPUT_DIR}")

args.config = Path(args.config)
if not args.config.is_absolute():
    args.config = Path.cwd() / args.config
if not args.config.exists():
    raise ValueError(f"Config file {args.config} does not exist")
sys.path.append(str(args.config.parent))
config = import_module(str(args.config.stem))

device = f"cuda:{args.device}" if torch.cuda.is_available() else "cpu"
print(f"Using device {device}")

#===================================================================================================
# Set up paths for data and results
#===================================================================================================
no_name = (args.name == 'unspecified')
BASE_INDEX = DATA_DIR / "index.csv"

CLEAN_SUFFIX = f"{'no_border_' if config.rm_border else ''}{'rm_' + str(config.remove_size)}"
CLEAN_INDEX = DATA_DIR / f"index_clean_{CLEAN_SUFFIX}.csv"

SHARP_SUFFIX = f"{config.sharpness_threshold}".split(".")[-1] if config.sharpness_threshold is not None else "none"
SHARP_INDEX = DATA_DIR / f"{CLEAN_INDEX.stem}_sharp_{SHARP_SUFFIX}.csv"

if config.norm_strategy in ['threshold', 'percentile']:
    NORM_SUFFIX = f"{config.norm_strategy}{f'_{config.norm_min}_{config.norm_max}' if config.norm_strategy in ['threshold', 'percentile'] else ''}"
elif config.norm_strategy == 'spline':
    if args.image_mask_cache or args.all:
        print("WARNING: automatically setting buckets to 2^8 for spline normalization")
        buckets = 2 ** 8
    else:
        from well_normalization import buckets
    NORM_SUFFIX = f"{config.norm_strategy}_{buckets}"
else:
    NORM_SUFFIX = config.norm_strategy
NORM_INDEX = DATA_DIR / f"{CLEAN_INDEX.stem if config.sharpness_threshold is None else SHARP_INDEX.stem}_norm_{NORM_SUFFIX}.csv"

NAME_INDEX = DATA_DIR / f"index_{args.name}.csv"
CONFIG_FILE = DATA_DIR / f"{args.name}.py"

try:
    sys.path.append(str(CONFIG_FILE.parent))
    dataset_config = None if no_name else import_module(str(CONFIG_FILE.stem))
except ModuleNotFoundError:
    if NAME_INDEX.exists():
        print(f"Config file {CONFIG_FILE} not found, but index file {NAME_INDEX} exists, this should not be the case")
    dataset_config = None

RGB_DATASET = DATA_DIR / f"rgb_{args.name}.pt"
EMBEDDINGS_DATASET = DATA_DIR / f"embeddings_{args.name}.pt"
GMM_PATH = DATA_DIR / f"gmm_{args.name}.pkl"
GMM_PROBS = DATA_DIR / f"gmm_probs_{args.name}.pt"

CHANNELS = load_channel_names(DATA_DIR) if config.channels is None else config.channels
if config.channels is None:
    save_channel_names(DATA_DIR, CHANNELS)
DAPI, TUBL, CALB2 = config.dapi, config.tubl, config.calb2

#===================================================================================================
# Implementation of Pipeline Steps
#===================================================================================================
if args.stats is not None:
    if args.stats == stats_opt[NAME]:
        # find all files like index_{name}.csv and print name
        print(f"Dataset names:")
        for file in DATA_DIR.glob("index_*.csv"):
            print(f"{file.stem[6:]}")

    data_paths_file, num_paths = create_image_paths_file(DATA_DIR, level=config.grouping)
    image_paths = image_paths_from_folders(data_paths_file)
    if args.stats == stats_opt[PIX_RANGE]:
        pixel_range_info(args, image_paths, CHANNELS, OUTPUT_DIR)
    if args.stats == stats_opt[NORM]:
        normalization_dry_run(args, config, image_paths, CHANNELS, OUTPUT_DIR, device)
    if args.stats == stats_opt[INT_IMG]:
        image_by_level_set(args, image_paths, CHANNELS, OUTPUT_DIR)
    if args.stats == stats_opt[SAMPLE]:
        from data_viz import save_image_grid, save_image
        assert not no_name, "Name of dataset must be specified"
        assert dataset_config is not None, "Dataset config file must be specified via name, this means that the config for this data doesn't exist or doesn't make the provided name"
        assert BASE_INDEX.exists() and NAME_INDEX.exists(), "Index files do not exist, run pipeline with at least until --single_cell"
        ORIGINAL_IMG = OUTPUT_DIR / "original_image.png"
        DATASET_IMG = OUTPUT_DIR / "dataset_cells.png"
        image_paths, _, _ = load_index_paths(BASE_INDEX)
        og_image_paths = image_paths[:args.viz_num]
        dataset_image_paths, _, _ = load_index_paths(NAME_INDEX)
        dataset_image_paths = dataset_image_paths[:args.viz_num]
        for i, (image_path, cell_images_path) in enumerate(zip(og_image_paths, dataset_image_paths)):
            image = torch.tensor(np.load(image_path).astype(np.float32)).squeeze()
            cell_images = torch.load(cell_images_path)
            img_file = ORIGINAL_IMG.with_name(f"{ORIGINAL_IMG.stem}_{i}{ORIGINAL_IMG.suffix}")
            cell_file = DATASET_IMG.with_name(f"{DATASET_IMG.stem}_{i}{DATASET_IMG.suffix}")
            save_image(image, img_file, cmaps=dataset_config.cmaps)
            save_image_grid(cell_images, cell_file, nrow=5, cmaps=dataset_config.cmaps)
    if args.stats == stats_opt[SHARP]:
        assert not no_name, "Name of dataset must be specified"
        assert dataset_config is not None, "Dataset config file must be specified via name, this means that the config for this data doesn't exist or doesn't make the provided name"
        assert NAME_INDEX.exists(), "Index files do not exist, run pipeline with at least until --single_cell"
        assert config.sharpness_threshold is not None, "Sharpness threshold must be specified in config file"
        dataset_image_paths, _, _ = load_index_paths(NAME_INDEX)
        dataset_image_paths = dataset_image_paths[:args.calc_num]
        sharpness_dry_run(dataset_image_paths, config.sharpness_threshold, OUTPUT_DIR, dataset_config.cmaps)

if args.image_mask_cache or args.all:
    print("Caching composite images and getting segmentation masks")
    data_paths_file, num_paths = create_image_paths_file(DATA_DIR, level=config.grouping, overwrite=args.rebuild)
    image_paths = image_paths_from_folders(data_paths_file)
    if BASE_INDEX.exists() and not args.rebuild:
        print("Index file already exists, skipping. Set --rebuild to overwrite.")
        save_channel_names(DATA_DIR, CHANNELS)
    else:
        multi_channel_model = True if CALB2 is not None else False
        segmentator = segmentator_setup(multi_channel_model, device)
        image_paths, nuclei_mask_paths, cell_mask_paths = get_masks(segmentator, image_paths, CHANNELS, DAPI, TUBL, CALB2, rebuild=args.rebuild)
        create_data_path_index(image_paths, cell_mask_paths, nuclei_mask_paths, BASE_INDEX, overwrite=True)
        save_channel_names(DATA_DIR, CHANNELS)

if args.clean_masks or args.all:
    if CLEAN_INDEX.exists() and not args.rebuild:
        print("Index file already exists, skipping. Set --rebuild to overwrite.")
    else:
        print("Cleaning masks")
        assert BASE_INDEX.exists(), "Index file does not exist, run --image_mask_cache first"
        image_paths, cell_mask_paths, nuclei_mask_paths = load_index_paths(BASE_INDEX)
        clean_cell_mask_paths, clean_nuclei_mask_paths, num_original, num_removed = clean_and_save_masks(cell_mask_paths, nuclei_mask_paths, CLEAN_SUFFIX,
            rm_border=config.rm_border, remove_size=config.remove_size)
        create_data_path_index(image_paths, clean_cell_mask_paths, clean_nuclei_mask_paths, CLEAN_INDEX, overwrite=True)
        print("Fraction removed:", num_removed / num_original)
        print("Total cells removed:", num_removed)
        print("Total cells remaining:", num_original - num_removed)

if args.filter_sharpness or args.all:
    assert CLEAN_INDEX.exists(), "Index file does not exist, run --clean_masks first"
    clean_image_paths, clean_cell_mask_paths, clean_nuclei_mask_paths = load_index_paths(CLEAN_INDEX)
    # we just overwrite the segmentation masks with the filtered ones so no need to get new paths
    sharp_cell_mask_paths, sharp_nuclei_mask_paths, num_removed, num_total = filter_masks_by_sharpness(clean_image_paths,
        clean_cell_mask_paths, clean_nuclei_mask_paths, config.sharpness_threshold, config.dapi, config.tubl, SHARP_SUFFIX,
        args.save_samples, config.cmaps)
    create_data_path_index(clean_image_paths, sharp_cell_mask_paths, sharp_nuclei_mask_paths, SHARP_INDEX, overwrite=True)
    print("Fraction blurry that were removed:", num_removed / num_total)

if args.normalize or args.all:
    import well_normalization as spline
    print("Normalizing images")
    if NORM_INDEX.exists() and not args.rebuild:
        print("Index file already exists, skipping. Set --rebuild to overwrite.")
    else:
        assert SHARP_INDEX.exists() or CLEAN_INDEX.exists(), "Index file does not exist, run --image_mask_cache (and optionally --clean_masks) first"
        if config.sharpness_threshold is not None:
            print("Using sharpness filtered images")
            SRC_INDEX = SHARP_INDEX
        else:
            print("Using cleaned images")
            SRC_INDEX = CLEAN_INDEX
        assert SRC_INDEX.exists(), "Index file does not exist, run --filter_sharpness (and optionally --clean_masks) first"
        assert config.norm_strategy is not None, "Normalization strategy not set in config"

        if config.norm_strategy == 'spline':
            if SRC_INDEX != spline.PRECALC_INDEX_PATH:
                raise NotImplementedError("Spline normalization requires precalculated well percentiles, run well_percentiles.ipynb and well_normalization.py first with your desired input data.")
            image_paths, cell_mask_paths, nuclei_mask_paths = load_index_paths(spline.PRECALC_INDEX_PATH)
            mask_paths = cell_mask_paths
            normalized_image_paths = []
            for i, (image_path, mask_path) in tqdm(enumerate(zip(image_paths, mask_paths)), total=len(image_paths), desc="Calculating well percentiles"):
                normalization_function = spline.well_normalization_map(spline.well_percentiles[i], spline.normalized_well_percentiles[i], range_max=(np.iinfo(np.uint16).max + 1))
                images = np.load(image_path).astype("float32")
                masks_path = str(mask_path) + ".npy"
                masks = np.load(masks_path)[:, None, ...].astype("float32")
                images = images * (masks > 0)
                images = images
                normalized_images = normalization_function(np.copy(images))
                new_image_path = image_path.parent / (image_path.stem + f"_spline_{spline.buckets}_normalized")
                np.save(new_image_path, normalized_images)
                new_image_path = new_image_path.parent / (new_image_path.stem + ".npy")
                normalized_image_paths.append(new_image_path)
            create_data_path_index(normalized_image_paths, cell_mask_paths, nuclei_mask_paths, NORM_INDEX, overwrite=True)
        else:
            image_paths, cell_mask_paths, nuclei_mask_paths = load_index_paths(SRC_INDEX)
            norm_paths = normalize_images(image_paths, cell_mask_paths, config.norm_strategy, config.norm_min, config.norm_max,
                NORM_SUFFIX, batch_size=100 if config.grouping == -1 else 1, save_samples=args.save_samples, cmaps=config.cmaps)
            create_data_path_index(norm_paths, cell_mask_paths, nuclei_mask_paths, NORM_INDEX, overwrite=True)

if args.single_cell or args.all:
    print("Cropping single cell images")
    assert not no_name, "Name of dataset must be specified"
    if NAME_INDEX.exists() and not args.rebuild:
        print("Index file already exists, skipping. Set --rebuild to overwrite.")
    else:
        assert NORM_INDEX.exists(), "Index file for normalized images does not exist, run --normalize first"
        print(NORM_INDEX)
        image_paths, cell_mask_paths, nuclei_mask_paths = load_index_paths(NORM_INDEX)
        seg_image_paths, clean_cell_mask_paths, clean_nuclei_mask_paths = crop_images(image_paths, cell_mask_paths, nuclei_mask_paths, config.cutoff, config.nuc_margin)
        final_image_paths, final_cell_mask_paths, final_nuclei_mask_paths = resize(seg_image_paths, clean_cell_mask_paths, clean_nuclei_mask_paths, config.output_image_size, args.name)
        create_data_path_index(final_image_paths, final_cell_mask_paths, final_nuclei_mask_paths, NAME_INDEX, overwrite=True)

        # save the source of the config module to the data directory with name args.name + '.py'
        # this will allow us to reproduce the results later
        with open(CONFIG_FILE, "w") as f:
            f.write("\n\n# Source of config module:\n")
            f.write(f"\n\n# Using normalized images from {NORM_INDEX}:\n")
            f.write(inspect.getsource(config))

        dataset_config = config

if args.rgb or args.all:
    from data import CellImageDataset, SimpleDataset
    assert not no_name, "Name of dataset must be specified"
    assert dataset_config is not None, "Dataset config file must be specified via name, this means that the config for this data doesn't exist or doesn't make the provided name"
    if SimpleDataset.has_cache_files(RGB_DATASET) and not args.rebuild:
        print("RGB images file already exists, skipping. Set --rebuild to overwrite.")
    else:
        print("Creating RGB images")
        assert NAME_INDEX.exists(), "Index file for single cell images does not exist, run --single_cell first"
        assert CONFIG_FILE.exists(), "Config file does not exist for the dataset, something might have gone wrong when you ran --single_cell"
        dataset = CellImageDataset(NAME_INDEX, dataset_config.cmaps, batch_size=args.batch_size)
        rgb_dataset = dataset.as_rgb()
        rgb_dataset.save(RGB_DATASET)

if args.dino_cls or args.dino_cls_ref or args.all:
    from data import CellImageDataset, SimpleDataset
    from models import DINO
    assert not (args.dino_cls and args.dino_cls_ref), "Cannot run both DINO classification and DINO classification with reference at the same time, please run separately."
    assert not no_name, "Name of dataset must be specified"
    assert dataset_config is not None, "Dataset config file must be specified via name, this means that the config for this data doesn't exist or doesn't make the provided name"

    if args.dino_cls_ref:
        EMBEDDINGS_DATASET = EMBEDDINGS_DATASET.parent / ("ref_" + EMBEDDINGS_DATASET.name)

    if EMBEDDINGS_DATASET.exists() and not args.rebuild:
        print("Embeddings file already exists, skipping. Set --rebuild to overwrite.")
    else:
        assert args.dino_cls_ref or SimpleDataset.has_cache_files(RGB_DATASET), "RGB dataset does not exist, run --rgb first"
        assert args.dino_cls or NAME_INDEX.exists(), "Index file for single cell images does not exist, run --single_cell first"
        print("Running DINO model to get embeddings")
        if type(dataset_config.output_image_size) != tuple:
            dataset_config.output_image_size = (dataset_config.output_image_size, dataset_config.output_image_size)
        dino = DINO(imsize=dataset_config.output_image_size).to(device)
        if args.dino_cls:
            dataset = SimpleDataset(path=RGB_DATASET)
        elif args.dino_cls_ref:
            dataset = CellImageDataset(NAME_INDEX, ["pure_blue", "pure_red"], channels=[0, 1])
            assert dataset[0].shape[0] == 2, "Dataset should've been only two channels at this point, got shape " + str(dataset[0].shape)
            dataset = dataset.as_rgb()
            assert dataset[0].shape[0] == 3, "Dataset should've been converted to RGB at this point, got shape " + str(dataset[0].shape)
            assert torch.sum(dataset[0:10, 1, :, :] != 0) == 0, "Dataset should've been converted to RGB with green channel zeroed out"
        dataloader = DataLoader(dataset, batch_size=args.batch_size, num_workers=1, shuffle=False)

        embeddings = []
        with torch.no_grad():
            for batch in tqdm(iter(dataloader), desc="Embedding images with DINOv2"):
                batch = batch.to(device)
                batch_embedding = dino(batch).cpu()
                embeddings.append(batch_embedding)
        embeddings = torch.cat(embeddings)

        torch.save(embeddings, EMBEDDINGS_DATASET)
        print(embeddings.shape)

if args.fucci_gmm or args.all:
    from sklearn.mixture import GaussianMixture
    from data import CellImageDataset, SimpleDataset
    assert not no_name, "Name of dataset must be specified"
    assert NAME_INDEX.exists(), "Index file for single cell images does not exist, run --single_cell first"
    if GMM_PROBS.exists() and not args.rebuild:
        print("GMM probabilities file already exists, skipping. Set --rebuild to overwrite.")
    else:
        dataset = CellImageDataset(NAME_INDEX)
        dataloader = DataLoader(dataset, batch_size=1000, num_workers=1, shuffle=False)
        FUCCI_intensities = []
        for batch in tqdm(iter(dataloader), desc="Getting FUCCI intensities"):
            FUCCI_intensities.append(torch.mean(batch[:, 2:], dim=(2, 3)))
        FUCCI_intensities = torch.cat(FUCCI_intensities)
        FUCCI_intensities = torch.log10(FUCCI_intensities + 1e-6)
        plt.clf()
        sns.kdeplot(x=FUCCI_intensities[:, 0], y=FUCCI_intensities[:, 1])
        plt.savefig(DATA_DIR / f"fucci_plot_{args.name}.png")
        plt.clf()

        print("Creating GMM")
        gmm = GaussianMixture(n_components=3)
        gmm.fit(FUCCI_intensities)
        pickle.dump(gmm, open(GMM_PATH, "wb"))
        print("Saved GMM to pickle file at " + str(GMM_PATH))

        print("Creating GMM probabilities")
        probs = gmm.predict_proba(FUCCI_intensities)
        probs = torch.tensor(probs)
        torch.save(probs, GMM_PROBS)
        print("Saved GMM probabilities to torch .pt file at " + str(GMM_PROBS))
        GMM_PLOT = OUTPUT_DIR / f"gmm_plot_{args.name}.png"
        plt.clf()
        sns.kdeplot(x=FUCCI_intensities[:, 0], y=FUCCI_intensities[:, 1], hue=probs.argmax(dim=1), palette="Set2")
        plt.savefig(GMM_PLOT)
        plt.clf()
        print("Saved GMM plot to " + str(GMM_PLOT))


import pytest
import pytest_check as check

from pages.home.login_page import HomePage
from utilities.BaseClass import BaseClass
from utilities.reusablemethods import CustomMethods

class Test_dropdown_validation(BaseClass):

    @pytest.mark.skip
    def test_dropdown_staticDropDown(self):
        log = self.getLogger()
        res = CustomMethods(self.driver)
        hPage = HomePage(self.driver)
        hPage.select_visibleText_In_DropDown("Option2")
        log.info("Selected the option Option 2")
        check.is_true(res.compare_two_strings(hPage.get_selected_option(), "Option2"))

    @pytest.mark.skip
    def test_allValues_In_DropDown(self):
        log = self.getLogger()
        res = CustomMethods(self.driver)
        hPage = HomePage(self.driver)
        count = hPage.print_all_Values_dropdown()
        log.info("checking the count of values in drop down")
        check.is_true(count == 4)

    @pytest.mark.skip
    def test_select_value_from_autosuggestion(self):
        log = self.getLogger()
        res = CustomMethods(self.driver)
        hPage = HomePage(self.driver)
        hPage.select_value_from_autocomplete("aus", "austria")
        text_selected = res.getText_from_textbox(hPage.autocomplete_locator, hPage.autocomplete_locatortype)
        log.info("Auto complete text --> "+text_selected)
        check.is_true(res.compare_two_strings(text_selected, "austria"))


from flask import render_template,request,redirect,url_for,abort
from flask_wtf import form
from . import main
from flask_login import login_required,current_user,login_user,logout_user
from .forms import UpdateProfile,BusinessForm,ReviewForm, SearchForm,LocationSearchForm
from .. import db,photos
from ..models import User,Business,Review
from app import models

#views
@main.route('/')
def index():
    search_bs = request.args.get("location")
    search_service = request.args.get("service")
    businessOnLocation = Business.query.filter_by(location=search_bs).all()
    businessByService = Business.query.filter_by(service=search_service).all()

    return render_template('index.html',businessOnLocation=businessOnLocation,businessByService=businessByService)

@main.route('/all')
def all():
    all_business = Business.query.filter_by().all()
    return render_template('all.html',all_business=all_business)

@main.route('/user/<uname>')
def profile(uname):
    user = User.query.filter_by(username = uname).first()
    business = Business.query.filter_by(user_id = current_user.id).all()
    if user is None:
        abort(404)

    return render_template("profile/profile.html", user = user, business=business)

@main.route('/user/<uname>/update',methods = ['GET','POST'])
@login_required
def update_profile(uname):
    user = User.query.filter_by(username = uname).first()
    if user is None:
        abort(404)

    form = UpdateProfile()

    if form.validate_on_submit():
        user.bio = form.bio.data

        db.session.add(user)
        db.session.commit()

        return redirect(url_for('.profile',uname=user.username))

    return render_template('profile/update.html',form =form)

@main.route('/user/<uname>/update/pic',methods= ['POST'])
@login_required
def update_pic(uname):
    user = User.query.filter_by(username = uname).first()
    if 'photo' in request.files:
        filename = photos.save(request.files['photo'])
        path = f'photos/{filename}'
        user.profile_pic_path = path
        db.session.commit()
    return redirect(url_for('main.profile',uname=uname))

@main.route('/user/<uname>/business',methods= ['POST','GET'])
@login_required
def upload_business(uname):
    user = User.query.filter_by(username = uname).first()
    form = BusinessForm()
    if user is None:
        abort(404)

    if form.validate_on_submit():
        businessname = form.businessname.data
        contact = form.contact.data
        service = form.service.data
        location= form.location.data
        website = form.website.data
        user_id = current_user.id

        business = Business(businessname=businessname,contact=contact,service=service,location=location,website=website,user_id=user_id)
        business.save_business()
        return redirect(url_for('main.all'))
    return render_template('upload_business.html',form=form,title='Add Business',legend='Add Business')

@main.route('/search', methods=['GET', 'POST'])
def search():
    searchform = SearchForm()
    if searchform.validate_on_submit():
        selectedservice = searchform.service.data
        businessByService = Business.query.filter_by(service=selectedservice).all()
        return render_template('results.html',businessByService=businessByService)
    return render_template('search.html',searchform=searchform,title='Search',legend='Add Business')

@main.route('/location',methods=['GET', 'POST'])
def location():
    search_bs = request.args.get("location")
    businessOnLocation = Business.query.filter_by(location=search_bs).all()
    return render_template('results.html',businessOnLocation=businessOnLocation)

# @main.route('/location',methods=['GET', 'POST'])
# def location():
#     locationform = LocationSearchForm()
#     if locationform.validate_on_submit():
#         selectedlocation = locationform.location.data
#         businessOnLocation = Business.query.filter_by(location=selectedlocation).all()
#         return render_template('results.html',businessOnLocation=businessOnLocation)
#     return render_template('location.html',locationform=locationform,title='Search',legend='Add Business')

# locationform = LocationSearchForm()
#     if locationform.validate_on_submit():
#         businesses = Business.query.filter_by().all()
#         for business in businesses:
#             if business.location == locationform.location.data:
#                 return redirect(url_for('main.index',business=business))
#     return render_template('search.html',locationform=locationform,title='Search',legend='Add Business')

@main.route("/review/<int:business_id>",methods=["POST","GET"])
@login_required
def reviews(business_id):
    form = ReviewForm()
    business = Business.query.get(business_id)
    all_reviews = Review.get_reviews(business_id)
    if form.validate_on_submit():
        new_review = form.review.data
        business_id = business_id
        user_id = current_user._get_current_object().id
        review_object = Review(review=new_review,user_id=user_id,business_id=business_id)
        review_object.save_review()
    return render_template("reviews.html",review_form = form,all_reviews = all_reviews,business = business)


import csv
import pandas as pd
import numpy as np

np.set_printoptions(threshold=np.inf)
path = ("/home/bunny/NewsAnalysisCode/")

Alphabets = ['A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z']

for filename in Alphabets:
    textLocation = path+"Words/Positive/"+filename+""
    csvLocation = path+"WordsCSV/Positive/"+filename+".csv"
    with open(textLocation,'r') as textfile:
        data = textfile.read()
        wordsList = data.split(",")
        with open(csvLocation,'w',newline='')as f:
            thewriter = csv.writer(f)
            thewriter.writerow(['Words', 'Value','Occurrences'])
            for word in wordsList:
                thewriter.writerow([''+word, '4','0'])
        print(filename+": Successful")


import asyncio
import logging
import os
import tempfile
import time
import unittest
from logging.handlers import RotatingFileHandler
from pathlib import Path

import numpy as np
import pytest
from lsst.daf.butler.registry.interfaces import DatabaseConflictError
from lsst.obs.lsst.translators.lsstCam import LsstCamTranslator
from lsst.ts import mtaos
from lsst.ts.wep.task.cutOutDonutsCwfsTask import CutOutDonutsCwfsTask
from lsst.ts.wep.utility import CamType
from lsst.ts.wep.utility import getModulePath as getModulePathWep

class TestUtility(unittest.TestCase):
    """Test the Utility functions."""

    def setUp(self):
        self.dataDir = tempfile.TemporaryDirectory(
            dir=mtaos.getModulePath().joinpath("tests").as_posix()
        )

    def tearDown(self):
        self.dataDir.cleanup()

    def testGetModulePath(self):
        modulePath = mtaos.getModulePath()
        self.assertTrue(modulePath.exists())
        self.assertTrue("ts_mtaos" in modulePath.name.lower())

    def testGetConfigDir(self):
        ansConfigDir = mtaos.getModulePath().joinpath("policy")
        self.assertEqual(mtaos.getConfigDir(), ansConfigDir)

    def testGetLogDir(self):
        ansLogDir = mtaos.getModulePath().joinpath("logs")
        self.assertEqual(mtaos.getLogDir(), ansLogDir)
        self.assertTrue(ansLogDir.exists())

    def testGetIsrDirPathNotAssigned(self):
        isrDir = mtaos.getIsrDirPath()
        self.assertEqual(isrDir, None)

    def testGetIsrDirPath(self):
        ISRDIRPATH = "/path/to/isr/dir"
        os.environ["ISRDIRPATH"] = ISRDIRPATH

        isrDir = mtaos.getIsrDirPath()
        self.assertEqual(isrDir, Path(ISRDIRPATH))

        os.environ.pop("ISRDIRPATH")

    def testGetCamType(self):
        self.assertEqual(mtaos.getCamType("lsstCam"), CamType.LsstCam)
        self.assertEqual(mtaos.getCamType("lsstFamCam"), CamType.LsstFamCam)
        self.assertEqual(mtaos.getCamType("comcam"), CamType.ComCam)

        self.assertRaises(ValueError, mtaos.getCamType, "wrongType")

    def testGetCscName(self):
        cscName = mtaos.getCscName()
        self.assertEqual(cscName, "MTAOS")

    def testAddRotFileHandler(self):
        log = logging.Logger("test")
        dataDirPath = self.dataDir.name
        filePath = Path(dataDirPath).joinpath("test.log")
        mtaos.addRotFileHandler(
            log, filePath, logging.DEBUG, maxBytes=1e3, backupCount=5
        )

        handlers = log.handlers
        self.assertEqual(len(handlers), 1)
        self.assertTrue(isinstance(handlers[0], RotatingFileHandler))

        for counter in range(20):
            log.critical("Test file rotation.")
            time.sleep(0.2)

        numOfFile = self._getNumOfFileInFolder(dataDirPath)
        self.assertEqual(numOfFile, 2)

    def _getNumOfFileInFolder(self, folder):
        items = Path(folder).glob("*")
        files = [aItem for aItem in items if aItem.is_file()]

        return len(files)

    def test_get_formatted_corner_wavefront_sensors_ids(self):
        mtaos_cwfs_detector_ids = set(
            [
                int(detector_id)
                for detector_id in mtaos.get_formatted_corner_wavefront_sensors_ids().split(
                    ","
                )
            ]
        )

        detector_mapping = LsstCamTranslator.detector_mapping()

        cwfs_task = CutOutDonutsCwfsTask()

        expected_cwfs_detector_ids = set(
            [
                detector_mapping[cwfs_detector_name][0]
                for cwfs_detector_name in cwfs_task.extraFocalNames
                + cwfs_task.intraFocalNames
            ]
        )

        assert mtaos_cwfs_detector_ids == expected_cwfs_detector_ids

    def test_timeit(self):
        @mtaos.timeit
        def my_retval(arg1, arg2, arg3, arg4, sleep_time, **kwargs):
            time.sleep(sleep_time)
            return arg1, arg2, arg3, arg4

        @mtaos.timeit
        async def amy_retval(arg1, arg2, arg3, arg4, sleep_time, **kwargs):
            await asyncio.sleep(sleep_time)
            return arg1, arg2, arg3, arg4

        exec_time = {}
        sleep_time = 0.1

        for i in range(10):
            r_a1, r_a2, r_a3, r_a4 = my_retval(
                arg1="this",
                arg2="is",
                arg3="a",
                arg4="test",
                sleep_time=sleep_time,
                log_time=exec_time,
            )
            self.assertEqual(r_a1, "this")
            self.assertEqual(r_a2, "is")
            self.assertEqual(r_a3, "a")
            self.assertEqual(r_a4, "test")

        for i in range(10):
            r_a1, r_a2, r_a3, r_a4 = asyncio.run(
                amy_retval(
                    arg1="this",
                    arg2="is",
                    arg3="a",
                    arg4="test",
                    sleep_time=sleep_time,
                    log_time=exec_time,
                )
            )
            self.assertEqual(r_a1, "this")
            self.assertEqual(r_a2, "is")
            self.assertEqual(r_a3, "a")
            self.assertEqual(r_a4, "test")

        self.assertTrue("MY_RETVAL" in exec_time)
        self.assertTrue("AMY_RETVAL" in exec_time)

        self.assertAlmostEqual(sleep_time, np.mean(exec_time["MY_RETVAL"]), 2)
        self.assertAlmostEqual(sleep_time, np.mean(exec_time["AMY_RETVAL"]), 2)

    @pytest.mark.xfail(
        reason="There is something wrong with the test data that causes this to fail.",
        raises=DatabaseConflictError,
    )
    def test_define_visit(self) -> None:
        data_path = os.path.join(
            getModulePathWep(), "tests", "testData", "gen3TestRepo"
        )

        mtaos.define_visit(
            data_path=data_path,
            collections=["LSSTCam/raw/all"],
            instrument_name="LSSTCam",
            exposures_str="exposure IN (4021123106001, 4021123106002)",
        )

if __name__ == "__main__":
    # Do the unit test
    unittest.main()


"""
https://leetcode-cn.com/problems/sorting-the-sentence/

一个 句子 指的是一个序列的单词用单个空格连接起来，且开头和结尾没有任何空格。每个单词都只包含小写或大写英文字母。

我们可以给一个句子添加 从 1 开始的单词位置索引 ，并且将句子中所有单词 打乱顺序 。
    比方说，句子 "This is a sentence" 可以被打乱顺序得到 "sentence4 a3 is2 This1" 或者 "is2 sentence4 This1 a3" 。

给你一个 打乱顺序 的句子 s ，它包含的单词不超过 9 个，请你重新构造并得到原本顺序的句子。

示例 1：
    输入：s = "is2 sentence4 This1 a3"
    输出："This is a sentence"
    解释：将 s 中的单词按照初始位置排序，得到 "This1 is2 a3 sentence4" ，然后删除数字。

示例 2：
    输入：s = "Myself2 Me1 I4 and3"
    输出："Me Myself and I"
    解释：将 s 中的单词按照初始位置排序，得到 "Me1 Myself2 and3 I4" ，然后删除数字。

提示：
    2 <= s.length <= 200
    s 只包含小写和大写英文字母、空格以及从 1 到 9 的数字。
    s 中单词数目为 1 到 9 个。
    s 中的单词由单个空格分隔。
    s 不包含任何前导或者后缀空格。

"""

class Solution:
    def sortSentence(self, s: str) -> str:
        words = s.split(" ")    # 分割字符串
        arr = ["" for _ in range(len(s))]   # 单词数组
        for ch in words:
            # 计算位置索引对应的单词数组下标，并将单词放入对应位置
            # 数组下标为 0 开头，位置索引为 1 开头
            arr[int(ch[-1])-1] = ch[:-1]
        return " ".join(arr)

class Solution:
    def sortSentence(self, s: str) -> str:
        words = s.split(" ")
        words = sorted(words, key = lambda w : w[-1])
        return ' '.join(w[:-1] for w in words)

if __name__ == "__main__":
    s = "is2 sentence4 This1 a3"
    sol = Solution()
    result = sol.sortSentence(s)
    print(result)


import pandas as pd
import numpy as np
import jieba

from WordMap import WordMap

neg=pd.read_excel('neg.xls',header=None,index=None)
pos=pd.read_excel('pos.xls',header=None,index=None)
pn=pd.concat([pos,neg],ignore_index=True)

cw = lambda x: list(jieba.cut(x, HMM=False))
pn['words'] = pn[0].apply(cw)

w = []
for i in pn['words']:
  w.extend(i)

wordmap = WordMap()
wordmap.build(w, 35000)
wordmap.save("word.map")


import os
from multiprocessing import shared_memory

def clearUp(share_name):
    try:
        shm_a = shared_memory.SharedMemory(name=share_name, create=False, size=268)
        shm_a.unlink()
    except Exception as e:
        print("delShareInfo:", e)
    path = os.path.join("/data/tmp/boost_interprocess", share_name)
    if os.path.exists(path):
        os.remove(path)


###  out_im: 4_dim trnsor   ####   cacu_loss
###  raw_im: 4_dim trnsor
###  bin_mask: 2_dim array
import tensorflow as tf
import functools

def get_gram(x):
    ba,hi,wi,ch = [i.value for i in x.get_shape()]
    feature = tf.reshape(x,[ba,int(hi*wi),ch])
    feature_T = tf.transpose(feature,[0,2,1])
    gram = tf.matmul(feature_T,feature)
    size = 1/(hi*wi*ch)
    return gram*size

def hole_loss(I_out,I_gt,bin_mask):
    L_hole = tf.reduce_mean(tf.abs((1-bin_mask)*(I_out-I_gt)))
    return L_hole

def valid_loss(I_out,I_gt,bin_mask):
    L_valid = tf.reduce_mean(tf.abs(bin_mask*(I_out-I_gt)))
    return L_valid

def Percept_loss(fai_out,fai_gt,fai_comp,layers):
    out_gt = []
    compt_gt = []
    for layer in layers:
        out_gt.append(tf.reduce_mean(tf.abs(fai_out[layer]-fai_gt[layer])))
        compt_gt.append(tf.reduce_mean(tf.abs(fai_comp[layer]-fai_gt[layer])))
    out_gt_loss = functools.reduce(tf.add,out_gt)
    compt_gt_loss = functools.reduce(tf.add,compt_gt)
    return out_gt_loss+compt_gt_loss

def Style_loss_out(fai_out,fai_gt,layers):
    styleloss = []
    for layer in layers:
        gram_out = get_gram(fai_out[layer])
        gram_gt = get_gram(fai_gt[layer])
        styleloss.append(tf.reduce_mean(tf.abs(gram_out-gram_gt)))
    style_out_loss = functools.reduce(tf.add,styleloss)
    return style_out_loss

def Style_loss_comp(fai_comp,fai_gt,layers):
    styleloss = []
    for layer in layers:
        gram_comp = get_gram(fai_comp[layer])
        gram_gt = get_gram(fai_gt[layer])
        styleloss.append(tf.reduce_mean(tf.abs(tf.subtract(gram_comp,gram_gt))))
    style_comp_loss = functools.reduce(tf.add,styleloss)
    return style_comp_loss

def Tv_loss(I_comp):
    I_comp_size = [i.value for i in I_comp.get_shape()]
    x_size = int(I_comp_size[2]*(I_comp_size[1]-1)*I_comp_size[3])
    y_size = int(I_comp_size[1]*(I_comp_size[2]-1)*I_comp_size[3])
    I_comp_x1 = I_comp[:,0:(I_comp_size[1]-1),:,:]
    I_comp_x2 = I_comp[:,1:I_comp_size[1],:,:]
    I_comp_y1 = I_comp[:,:,0:(I_comp_size[2]-1),:]
    I_comp_y2 = I_comp[:,:,1:I_comp_size[2],:]
    tv_x = tf.reduce_sum(tf.squared_difference(I_comp_x1,I_comp_x2))/x_size
    tv_y = tf.reduce_sum(tf.squared_difference(I_comp_y1,I_comp_y2))/y_size
    tv_loss = tv_y+tv_x
    return tv_loss

def get_total_loss(I_out,I_gt,bin_mask,fai_out,fai_gt,fai_comp,layers,I_comp):
    l_hole = hole_loss(I_out,I_gt,bin_mask)
    l_valid = valid_loss(I_out,I_gt,bin_mask)
    percept_loss = Percept_loss(fai_out,fai_gt,fai_comp,layers)
    style_loss_out = Style_loss_out(fai_out,fai_gt,layers)
    style_loss_comp = Style_loss_comp(fai_comp,fai_gt,layers)
    tv_loss = Tv_loss(I_comp)
    all_loss = 6*l_hole + l_valid + 0.05*percept_loss + 120*(style_loss_out + style_loss_comp) + 0.1*tv_loss
    return all_loss,l_hole,l_valid,percept_loss,style_loss_out,style_loss_comp,tv_loss


import requests
from parsel import Selector

headers = {
    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
    "accept-language": "en-IN,en-GB;q=0.9,en-US;q=0.8,en;q=0.7",
    "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36",
}
url = "https://www.edeka24.de/"

def url_links(selector, xpath_expression, page=''):
    links = selector.xpath(xpath_expression).extract()
    repeated = set()
    for i in range(len(links)):
        if not links[i].startswith("https://www.edeka24.de/"):
            links[i] = "https://www.edeka24.de" + links[i]

    for link in links:
        urls = link.split('?')[0]
        if urls in repeated:
            break
        repeated.add(urls)
        url1=urls + "?pgNr=" + str(page)
        print(url1)
        response1 = requests.get(url1, headers=headers)
        selector1= Selector(response1.text)
        title =selector1.xpath("//div[@class='product-details']/a[@class='title']/h2/text()").extract()
        product_title = '\n'.join(title)
        print(product_title)
        # price =selector.xpath("//div[@class='left']/text()").extract()
        # product_price = '\n'.join(price)
        # print(product_price)
        # else:
        #     price=selector.xpath("//div[@class='product-details']//div[@class='price salesprice']/text()").extract()
        #     product_price = '\n'.join(price)
        #     print(product_price)
        # response1 = requests.get(url1, headers=headers)
        # selector1= Selector(response1.text)
        # title =selector1.xpath("//div[@class='product-details']/a[@class='title']/h2/text()").extract()
        # product_title = '\n'.join(title)
        # print(product_title)
        images =selector1.xpath("//div[@class='product-image']/a/img/@src").extract()
        image = '\n'.join(images)
        print(image)
        print()
def main_list(page=1):
    response = requests.get(url, headers=headers)
    print(response.status_code)
    selector = Selector(response.text)
    url_links(selector, '//a/@data-link', page)
    url_links(selector, "//li[@class='nav-item-mobile is-level-2']/a/@href", page)
    url_links(selector, '//li[@class="nav-item-mobile is-level-1"]/span/a/@href', page)
    main_list(page + 1)

main_list(1)


import torch
from deepsnap.graph import Graph
from deepsnap.hetero_graph import HeteroGraph
from typing import (
    Callable,
    Dict,
    List
)

class Batch(Graph):
    r"""
    A plain old python object modeling a batch of
    :class:`deepsnap.graph.Graph` objects as one big (disconnected) graph,
    with :class:`torch_geometric.data.Data` being the
    base class that all its methods can also be used here.
    In addition, graphs can be reconstructed via the assignment vector
    :obj:`batch`, which maps each node to its respective graph identifier.

    .. note::

        For more detailed use of :class:`deepsnap.batch.Batch`, see the `examples
        <https://github.com/snap-stanford/deepsnap/tree/master/examples>`_ folder.

    """
    def __init__(self, batch=None, **kwargs):
        super(Batch, self).__init__(**kwargs)

        self.batch = batch
        self.__data_class__ = Graph
        self.__slices__ = None

    @staticmethod
    def collate(follow_batch=[], transform=None, **kwargs):
        return lambda batch: Batch.from_data_list(
            batch, follow_batch, transform, **kwargs
        )

    @staticmethod
    def from_data_list(
        data_list: List[Graph],
        follow_batch: List = None,
        transform: Callable = None,
        **kwargs
    ):
        r"""
        Constructs A :class:`deepsnap.batch.Batch` object from a python list
        holding :class:`torch_geometric.data.Data` objects.
        The assignment vector :obj:`batch` is created on the fly.
        Additionally, creates assignment batch vectors for each key in
        :obj:`follow_batch`.

        Args:
            data_list (list): A list of :class:`deepsnap.graph.Graph` objects.
            follow_batch (list): Creates assignment batch vectors
                for each key.
            transform (callable): If it is not `None`, apply transform
                when batching.
            **kwargs: Other parameters.
        """
        if follow_batch is None:
            follow_batch = []
        if transform is not None:
            data_list = [
                data.apply_transform(
                    transform,
                    deep_copy=True,
                    **kwargs,
                )
                for data in data_list
            ]
        keys = [set(data.keys) for data in data_list]
        keys = list(set.union(*keys))
        assert "batch" not in keys

        batch, cumsum = Batch._init_batch_fields(keys, follow_batch)
        batch.__data_class__ = data_list[0].__class__
        batch.batch = []
        for i, data in enumerate(data_list):
            # Note: in heterogeneous graph, __inc__ logic is different
            Batch._collate_dict(
                data, cumsum,
                batch.__slices__, batch,
                data, follow_batch, i=i
            )
            if isinstance(data, Graph):
                if isinstance(data, HeteroGraph):
                    num_nodes = sum(data.num_nodes().values())
                else:
                    num_nodes = data.num_nodes
            else:
                raise TypeError(
                    "element in self.graphs of unexpected type"
                )
            if num_nodes is not None:
                item = torch.full((num_nodes, ), i, dtype=torch.long)
                batch.batch.append(item)

        if num_nodes is None:
            batch.batch = None

        Batch._dict_list_to_tensor(batch, data_list[0])

        return batch.contiguous()

    @staticmethod
    def _init_batch_fields(keys, follow_batch):
        batch = Batch()
        batch.__slices__ = {key: [0] for key in keys}

        for key in keys:
            batch[key] = []

        for key in follow_batch:
            batch[f"{key}_batch"] = []

        cumsum = {key: 0 for key in keys}
        return batch, cumsum

    @staticmethod
    def _collate_dict(
        curr_dict,
        cumsum: Dict[str, int],
        slices,
        batched_dict,
        graph,
        follow_batch,
        i=None
    ):
        r""" Called in from_data_list to collate a dictionary.
        This can also be applied to Graph object, since it has support for
        keys and __getitem__().

        Args:
            curr_dict: current dictionary to be added to the
                collated dictionary.
            cumsum: cumulative sum to be used for indexing.
            slices: a dictionary of the same structure as batched_dict,
                slices[key] indicates the indices to slice batch[key] into
                tensors for all graphs in the batch.
            batched_dict: the batched dictionary of the same structure
                as curr_dict. But all graph data are batched together.
        """
        if isinstance(curr_dict, dict):
            keys = curr_dict.keys()
        else:
            keys = curr_dict.keys
        for key in keys:
            item = curr_dict[key]
            if isinstance(item, dict):
                # recursively collate every key in the dictionary
                if isinstance(batched_dict[key], list):
                    # nested dictionary not initialized yet
                    assert len(batched_dict[key]) == 0
                    # initialize the nested dictionary for batch
                    cumsum[key] = {inner_key: 0 for inner_key in item.keys()}
                    slices[key] = {inner_key: [0] for inner_key in item.keys()}
                    batched_dict[key] = {}
                    for inner_key in item.keys():
                        batched_dict[key][inner_key] = []
                    for inner_key in follow_batch:
                        batched_dict[key][f"{key}_batch"] = []
                Batch._collate_dict(
                    item, cumsum[key],
                    slices[key], batched_dict[key],
                    graph, follow_batch, i=i
                )
                continue
            if torch.is_tensor(item) and item.dtype != torch.bool:
                item = item + cumsum[key]
            if torch.is_tensor(item):
                size = item.size(graph.__cat_dim__(key, curr_dict[key]))
            else:
                size = 1
            slices[key].append(size + slices[key][-1])
            cumsum[key] = cumsum[key] + graph.__inc__(key, item)
            batched_dict[key].append(item)

            if key in follow_batch:
                item = torch.full((size, ), i, dtype=torch.long)
                batched_dict[f"{key}_batch"].append(item)

    @staticmethod
    def _dict_list_to_tensor(dict_of_list, graph):
        r"""Convert a dict/Graph with list as values to a dict/Graph with
        concatenated/stacked tensor as values.
        """
        if isinstance(dict_of_list, dict):
            keys = dict_of_list.keys()
        else:
            keys = dict_of_list.keys
        for key in keys:
            if isinstance(dict_of_list[key], dict):
                # recursively convert the dictionary of list to dict of tensor
                Batch._dict_list_to_tensor(dict_of_list[key], graph)
                continue
            item = dict_of_list[key][0]
            if torch.is_tensor(item):
                if (
                    Graph._is_graph_attribute(key)
                    and item.ndim == 1
                    and (not item.dtype == torch.long)
                    and "feature" in key
                ):
                    # special consideration: 1D tensor for graph
                    # attribute (classification)
                    # named as: "graph_xx_feature"
                    # batch by stacking the first dim
                    dict_of_list[key] = torch.stack(
                        dict_of_list[key],
                        dim=0
                    )
                else:
                    # concat at the __cat_dim__
                    dict_of_list[key] = torch.cat(
                        dict_of_list[key],
                        dim=graph.__cat_dim__(key, item)
                    )
            elif isinstance(item, (float, int)):
                dict_of_list[key] = torch.tensor(dict_of_list[key])

    def to_data_list(self):
        r"""
        Reconstructs the list of :class:`torch_geometric.data.Data` objects
        from the batch object.
        The batch object must have been created via :meth:`from_data_list` in
        order to be able to reconstruct the initial objects.
        """
        if self.__slices__ is None:
            raise RuntimeError(
                "Cannot reconstruct data list from batch because the "
                "batch object was not created using Batch.from_data_list()"
            )

        keys = [key for key in self.keys if key[-5:] != "batch"]
        cumsum = {key: 0 for key in keys}
        data_list = []
        for i in range(len(self.__slices__[keys[0]]) - 1):
            # i: from 0 up to num graphs in the batch
            data = self.__data_class__()
            self._reconstruct_dict(
                i, keys, data, cumsum, self.__slices__, self, data
            )
            data_list.append(data)

        return data_list

    def _reconstruct_dict(
            self, graph_idx: int, keys, data_dict,
            cumsum: Dict[str, int], slices, batched_dict, graph):

        for key in keys:
            if isinstance(batched_dict[key], dict):
                # recursively unbatch the dict
                data_dict[key] = {}
                inner_keys = [
                    inner_key
                    for inner_key in batched_dict[key].keys()
                    if inner_key[-5:] != "batch"
                ]
                inner_cumsum = {inner_key: 0 for inner_key in inner_keys}
                inner_slices = slices[key]
                self._reconstruct_dict(
                    graph_idx, inner_keys,
                    data_dict[key], inner_cumsum,
                    inner_slices, batched_dict[key], graph
                )
                continue

            if torch.is_tensor(batched_dict[key]):
                data_dict[key] = batched_dict[key].narrow(
                    graph.__cat_dim__(key, batched_dict[key]),
                    slices[key][graph_idx],
                    slices[key][graph_idx + 1] - slices[key][graph_idx]
                )
                if batched_dict[key].dtype != torch.bool:
                    data_dict[key] = data_dict[key] - cumsum[key]
            else:
                data_dict[key] = (
                    batched_dict[key][
                        slices[key][graph_idx]:slices[key][graph_idx + 1]
                    ]
                )
            cumsum[key] = cumsum[key] + graph.__inc__(key, data_dict[key])

    @property
    def num_graphs(self) -> int:
        r"""
        Returns the number of graphs in the batch.

        Returns:
            int: The number of graphs in the batch.
        """
        return self.batch[-1].item() + 1

    def apply_transform(
        self,
        transform,
        update_tensor: bool = True,
        update_graph: bool = False,
        deep_copy: bool = False,
        **kwargs
    ):
        r"""
        Applies a transformation to each graph object in parallel by first
        calling `to_data_list`, applying the transform, and then perform
        re-batching again to a `Batch`.
        A transform should edit the graph object,
        including changing the graph structure, or adding
        node / edge / graph level attributes.
        The rest are automatically handled by the
        :class:`deepsnap.graph.Graph` object, including everything
        ended with `index`.

        Args:
            transform (callable): Transformation function applied to each graph object.
            update_tensor (bool): Whether use nx graph to update tensor attributes.
            update_graph (bool): Whether use tensor attributes to update nx graphs.
            deep_copy (bool): :obj:`True` if a new deep copy of batch is returned.
                This option allows modifying the batch of graphs without
                changing the graphs in the original dataset.
            kwargs: Parameters used in the transform function for each
                :class:`deepsnap.graph.Graph`.

        Returns:
            A batch object containing all transformed graph objects.

        """
        # TODO: transductive setting, assert update_tensor == True
        return self.from_data_list(
            [
                Graph(graph).apply_transform(
                    transform, update_tensor, update_graph, deep_copy, **kwargs
                )
                for graph in self.G
            ]
        )

    def apply_transform_multi(
        self,
        transform,
        update_tensors: bool = True,
        update_graphs: bool = False,
        deep_copy: bool = False,
        **kwargs
    ):
        r"""
        Compared to :meth:`apply_transform`, this allows multiple graph objects
        to be returned by the given transform function.

        Args:
            transform (callable): (Multiple return value) tranformation function
                applied to each graph object. It needs to return a tuple of
                Graph objects.
            update_tensors (bool): Whether use nx graph to update tensor attributes.
            update_graphs (bool): Whether use tensor attributes to update nx graphs.
            deep_copy (bool): :obj:`True` if a new deep copy of batch is returned.
                This option allows modifying the batch of graphs without
                changing the graphs in the original dataset.
            kwargs: Parameters used in the transform function for each
                :class:`deepsnap.graph.Graph`.

        Returns:
            A tuple of batch objects. The i-th batch object contains the i-th
            return value of the transform function applied to all graphs
            in the batch.
        """
        g_lists = (
            zip(
                *[
                    Graph(graph).apply_transform_multi(
                        transform, update_tensors, update_graphs,
                        deep_copy, **kwargs,
                    )
                    for graph in self.G
                ]
            )
        )
        return (self.from_data_list(g_list) for g_list in g_lists)