## IMPORT LIBRARIES
import pandas as pd
import geopandas as gpd
import pandana as pdn
import geopy
from geopy import Nominatim
from shapely.geometry import LineString
from sklearn.preprocessing import minmax_scale

def getRoutes(JSON_month, JSON_start, JSON_end):

    ##### ---------- LOAD PREPROCESSED DATA ---------- #####
    # DF ALREADY PROJECTED IN CRS 3857 - READ DOCUMENTATION TO ENLARGED THIS STEP

    ## 1/ LOAD NETWORK AS NX GRAPH
    filepath_nodes = './load/nodes.shp'
    filepath_edges = './load/edges.shp'
    nodes = gpd.read_file(filepath_nodes)
    edges = gpd.read_file(filepath_edges)

    ## 2/ LOAD TREES DATASET
    filepath_trees = './load/trees_proj.shp'
    tr_proj = gpd.read_file(filepath_trees)

    ## 3/ LOAD BLOOM DATASET
    filepath_bloom = './load/FLOWERING_SEV_trees.csv'
    bloom_df = pd.read_csv(filepath_bloom)

    ## 4/ DEFINE NETWORK CRS
    crs_proj = 3857

    ##### ---------- CREATE MONTH FILTER ---------- #####
    # encode the column to mask
    season = JSON_month
    ## FILTER TREE DATASET PER MONTH
    # display mask
    mask = bloom_df[season].isin([1.0])
    # PD series to PD DF
    bloom_mask = bloom_df[mask].copy()
    # clean DF
    bloom_cl = bloom_mask[['NAME', season]].copy()
    # join trees DF and filtered DF on family column
    tr_join = pd.merge(tr_proj, bloom_cl, how='left', on='NAME')
    # filter trees DF by dropping NAN rows
    tr_season = tr_join.dropna()
    # ---> OUTPUT 01: save seasonal trees as json
    tr_season_geojson = tr_season.to_json()

    ##### ---------- DEFINE ORIGIN & DESTINY ---------- #####
    ## DEFINE REVERSE GEOCODE FUNCTION
    def get_location_data(lat, lon):
        locator = Nominatim(user_agent="gds4ae")
        coords = (lat, lon)
        location = locator.reverse(coords)

        return location.address

    ## DEFINE PATH LOCATIONS
    geopy.geocoders.options.default_user_agent = "gds4ae"

    # source / origin
    split_coord_start = [j.split() for j in JSON_start.split(',') if j!='']
    split_coord_start = [item for sublist in split_coord_start for item in sublist]
    split_coord_start = [float(i) for i in split_coord_start]
    lat_coord_start = split_coord_start[0::2]
    lon_coord_start = split_coord_start[1::2]

    JSON_start_x = float(lon_coord_start[0])
    JSON_start_y = float(lat_coord_start[0])

    start_address = get_location_data(JSON_start_x, JSON_start_y)
    orig_address = gpd.tools.geocode(start_address, geopy.Nominatim).to_crs(crs_proj)

    # target / destiny
    split_coord_end = [j.split() for j in JSON_end.split(',') if j!='']
    split_coord_end = [item for sublist in split_coord_end for item in sublist]
    split_coord_end = [float(i) for i in split_coord_end]
    lat_coord_end = split_coord_end[0::2]
    lon_coord_end = split_coord_end[1::2]

    JSON_end_x = float(lat_coord_end[0])
    JSON_end_y = float(lon_coord_end[0])

    end_address = get_location_data(JSON_end_y, JSON_end_x)
    dest_address = gpd.tools.geocode(end_address, geopy.Nominatim).to_crs(crs_proj)

    # ---> OUTPUT 04: save directions as json
    orig_address_geojson = orig_address.to_json()
    dest_address_geojson = dest_address.to_json()

    ##### ---------- PANDANA ROUTE FUNCTION ---------- #####
    # BUILD LINESTRING FROM NODES MANUALLY
    def route_nodes_to_line(nodes, network):
        pts = network.nodes_df.loc[nodes, :]
        s = gpd.GeoDataFrame(
            {"src_node": [nodes[0]], "tgt_node": [nodes[1]]},
            geometry=[LineString(pts.values)],
            crs=crs_proj
        )
        return s

    ##### ---------- PANDANA SHORTEST ROUTE ---------- #####
    # create PDN graph to calculate the SHORTEST ROUTE based on EDGES LENGTH
    streets_pdn = pdn.Network(
                nodes.geometry.x,
                nodes.geometry.y,
                edges["node_start"],
                edges["node_end"],
                edges[["mm_len"]]
    )
    # snap A & B points to closest node on graph
    pt_nodes = streets_pdn.get_node_ids(
        [orig_address.geometry.x, dest_address.geometry.x],
        [orig_address.geometry.y, dest_address.geometry.y]
    )
    # route the shortest path
    route_nodes = streets_pdn.shortest_path(pt_nodes[0], pt_nodes[1])
    # build shortest route linestring
    short_path = route_nodes_to_line(route_nodes, streets_pdn)
    # get route length in KM
    short_path_len = streets_pdn.shortest_path_length(
        pt_nodes[0], pt_nodes[1]
    )
    short_path_km = round(short_path_len / 1000, 3) # Dist in Km, 3 digits
    # ---> OUTPUT 02: save seasonal route as json
    short_path_geojson = short_path.to_json()

    ##### ---------- PANDANA SEASONAL ROUTE ---------- #####
    # weight the seasonal path / the bigger the number, the higher priority to AMOUNT OF TREES Vs. DISTANCE
    power = 1000

    ## PREPROCESSING
    # snap trees to PDN network - get closest node IDS
    tr_nodes = tr_season.copy()
    tr_nodes['node_ids'] = streets_pdn.get_node_ids(tr_nodes['lon'], tr_nodes['lat'])
    # create PD series to count node occurrences and convert to PD DF
    tr_count = tr_nodes['node_ids'].value_counts()
    tr_count_df = pd.DataFrame(tr_count)
    # clean DF
    tr_count_df = tr_count_df.rename(columns={'node_ids': 'trees'})
    # create node ID column from index
    tr_count_df.reset_index(inplace=True)
    # duplicate count DF as node start
    tr_count_st = tr_count_df.copy()
    tr_count_st = tr_count_st.rename(columns={'index': 'node_start', 'trees': 'trees_start'})
    # duplicate count DF as node end
    tr_count_end = tr_count_df.copy()
    tr_count_end = tr_count_end.rename(columns={'index': 'node_end', 'trees': 'trees_end'})
    # join PDN graph edges and trees occurrences DF
    w_edges = pd.merge(edges, tr_count_st, how='left', on='node_start')
    w_edges = pd.merge(w_edges, tr_count_end, how='left', on='node_end')
    # add column to PDN EDGES with the total amount of trees per edge
    w_edges['trees_total']= w_edges['trees_start'] + w_edges['trees_end']
    # replace NAN values with 0
    w_edges['trees_total'].fillna(0, inplace=True)
    # attach to edges a scaled version of "mm_len"
    w_edges["scaled_len"] = minmax_scale(w_edges["mm_len"])
    # attach to edges a scaled negative version of the amount of trees / a street with more trees snapped will have a "shortest length"
    w_edges["scaled_trees"] = minmax_scale(-w_edges["trees_total"])
    # define the weighted column for routes calcultion
    w_edges["tr_len"] = w_edges["length"] * (w_edges["scaled_trees"]**power)

    # CALCULATE SEASONAL ROUTE
    # create PDN graph to calculate the SEASONAL ROUTE based on AM. OF TREES
    season_pdn = pdn.Network(
                        nodes.geometry.x,
                        nodes.geometry.y,
                        w_edges["node_start"],
                        w_edges["node_end"],
                        w_edges[["tr_len"]]
    )
    # snap A & B points to closest node on graph
    pt_tr_nodes = season_pdn.get_node_ids(
        [orig_address.geometry.x, dest_address.geometry.x],
        [orig_address.geometry.y, dest_address.geometry.y]
    )
    # route the seasonal path
    w_route_nodes = season_pdn.shortest_path(pt_tr_nodes[0], pt_tr_nodes[1])
    # build seasonal route linestring
    season_path = route_nodes_to_line(w_route_nodes, season_pdn)
    # ---> OUTPUT 03: save seasonal route as json
    season_path_geojson = season_path.to_json()

    #### return to APP.PY
    return season_path_geojson, short_path_geojson, tr_season_geojson, orig_address_geojson, dest_address_geojson


import uuid
from datetime import datetime

from backend.utils.utils import select

class RecordsStorage:
    def __init__(self):
        self.records = []

    def add(self, record_data, users):
        record = {
            "record_id": uuid.uuid4().int,
            "record_time": datetime.now(),
            **record_data
        }
        if "record_currency" not in record:
            record["record_currency"] = select(
                users, "user_id", record["user_id"]
            )[0]["user_currency"]
        self.records.append(record)
        return record

    def get_records_by_user_categories(self, user_id, category_id):
        if user_id == None:
            return self.records
        selected = select(self.records, "user_id", user_id)
        if category_id:
            selected = select(selected, "category_id", category_id)
        return selected

    def get_record_by_id(self, record_id):
        return select(self.records, "record_id", record_id)


import os
from aiogram import types
from aiogram.dispatcher import FSMContext
from docgen.creation.new_image import docMaking
from docgen.validate import validate
from states import MyStatesGroup
from keyboards import *

# докген начало
async def imgStart(msg: types.Message, state: FSMContext):
    example = '`Иван Иванов 24-08-1998 М UA +`'
    await msg.answer(text=f'Пришлите данные в формате: Имя Фамилия дд\-мм\-гггг пол\(м/ж\) страна\(UA/KZ/USA\) делать ли метаданные\(\+/\-\)\nПример: {example}',
                    parse_mode='MarkdownV2', reply_markup=setImgStartKb()
                     )
    await MyStatesGroup.imgReady.set()

# докген генерация и выдача
async def imgReady(msg: types.Message, state: FSMContext):
    imgData = await state.get_data()

    # проверяем есть ли уже данные, отличаются ли
    # новые данные от старых и была ли ошибка
    try:
        if msg.text == imgData['imgData']:
            pass
        else:
            if not (msg.text == 'Еще'):
                await state.update_data(imgData=msg.text)
    except:
        await state.update_data(imgData=msg.text)
    imgData = await state.get_data()

    # генерация изображения c ответом и обработка ошибок
    validatedData = validate(imgData['imgData'].split())
    if validatedData:
        waiting = await msg.answer(text='\u231B_Изображение генерируется\.\.\._', parse_mode='MarkdownV2', reply_markup=setImgReadyKb())
        doc = docMaking(validatedData)
        photo = types.InputFile(doc)
        await msg.answer_document(photo)
        await waiting.delete()
        await msg.answer(text='\u2705Готово!\nДля изображения с этими же данными, нажмите кнопку «Еще», или пришлите другие данные.',
                         reply_markup=setImgReadyKb())
        os.remove(doc)
        await MyStatesGroup.imgReady.set()
    else:
        await state.reset_state()
        await MyStatesGroup.imgReady.set()
        important = ('*соблюдайте пробелы и формат данных*')
        example = '`Иван Иванов 24-08-1998 М UA +`'
        await msg.answer(text=f'\u274C Данные введены некорректно, {important} как в примере:\n{example}',
                         parse_mode='MarkdownV2', reply_markup=setImgReadyKb())


import cv2

image = cv2.imread('images/people1.jpg')

image_gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
detector_facil = cv2.CascadeClassifier('Cascades/haarcascade_frontalface_default.xml')
detections = detector_facil.detectMultiScale(image_gray, scaleFactor=1.7)  # ele aumenta a escala, então precisa ser ajustado para obter menos falsos positivos
for (x, y, w, h) in detections:
    cv2.rectangle(image, (x, y), (x + w, y + h), (255, 0, 0), 2)

cv2.imshow('img', image)
cv2.waitKey(0)
cv2.destroyAllWindows()


from .views import *
from django.contrib.auth import views as auth_views
from django.contrib.auth.models import AnonymousUser
from datetime import timedelta, date
import code;

class CustomerIndexView(View):
    def get(self, request, *args, **kwargs):
        context = {}
        user = request.user
        featured_products = ProductVariant.objects.filter(is_selling=True, is_feature=True)[:6]
        for product in featured_products:
            options = VariantOption.objects.filter(product_id=product.id)
            setattr(product,"option", options.first().id)
        electronic_category = Category.objects.get(pk=1)
        electronic_subcategories = SubCategory.objects.filter(parent=electronic_category)
        for subcategory in electronic_subcategories:
            products = ProductVariant.objects.distinct().filter(is_selling=True,product__sub_category__id=subcategory.id)
            for product in products:
                options = VariantOption.objects.filter(product_id=product.id)
                setattr(product,"option", options.first().id)
            setattr(subcategory, "products", products)

        clothing_category = Category.objects.get(pk=3)
        clothing_subcategories = SubCategory.objects.filter(parent=clothing_category)
        for subcategory in clothing_subcategories:
            products = ProductVariant.objects.distinct().filter(is_selling=True,product__sub_category__id=subcategory.id)
            for product in products:
                options = VariantOption.objects.filter(product_id=product.id)
                setattr(product,"option", options.first().id)
            setattr(subcategory, "products", products)

        book_category = Category.objects.get(pk=2)
        book_subcategories = SubCategory.objects.filter(parent=book_category)
        for subcategory in book_subcategories:
            products = ProductVariant.objects.distinct().filter(is_selling=True,product__sub_category__id=subcategory.id)
            for product in products:
                options = VariantOption.objects.filter(product_id=product.id)
                setattr(product,"option", options.first().id)
            setattr(subcategory, "products", products)

        if 'cart' not in request.session:
            cart = []
        else:
            cart = request.session['cart']
        if user.is_authenticated:
            context["user_id"] = user
            context["favorite"] = Favorite.objects.filter(customer=user).count()
        context["featured_products"] = featured_products
        context["electronic_subcategories"] = electronic_subcategories
        context["clothing_subcategories"] = clothing_subcategories
        context["book_subcategories"] = book_subcategories

        context["cart"] = len(cart)
        return render(request, "boec_core/customer/index.html",context)

class CartView(View):
    def get(self, request, *args, **kwargs):
        context = {}
        user = request.user
        context["user"] = user
        cart = {}
        cart_items = []
        total = 0
        if 'cart' not in request.session:
            cart = {}
        else:
            cart = request.session['cart']
        for item in cart:
            variant = VariantOption.objects.get(pk=item)
            quantity = cart[item]
            amount = quantity * variant.price
            setattr(variant, 'quantity', quantity)
            setattr(variant, 'total', amount)
            total += amount
            cart_items.append(variant)
        context["cart"] = len(cart)
        context["cart_items"] = cart_items
        context["total"] = total

        return render(request, "boec_core/customer/shoping-cart.html",context)

class FavoriteView(View):
    def get(self, request, *args, **kwargs):
        context = {}
        user = request.user
        favorites = Favorite.objects.filter(customer=user)
        featured_products = []
        for favorite in favorites:
            featured_products.append(favorite.product)

        if 'cart' not in request.session:
            cart = []
        else:
            cart = request.session['cart']
        if user.is_authenticated:
            context["user_id"] = user
            context["favorite"] = Favorite.objects.filter(customer=user).count()
        context["featured_products"] = featured_products
        context["cart"] = len(cart)
        return render(request, "boec_core/customer/favorite.html",context)

class ShopGridView(View):
    def get(self, request, *args, **kwargs):
        context = {}
        user = request.user
        search = request.GET.get("search")
        if search is None:
            selling_products = ProductVariant.objects.filter(is_selling=True)
        else:
            selling_products = ProductVariant.objects.distinct().filter(is_selling=True, product__name__icontains=search)

        if 'cart' not in request.session:
            cart = []
        else:
            cart = request.session['cart']
        if user.is_authenticated:
            favorites = Favorite.objects.filter(customer=user)
            context["user_id"] = user
            context["favorite"] = Favorite.objects.filter(customer=user).count()
        context["selling_products"] = selling_products
        context["cart"] = len(cart)
        page_size = settings.PAGE_SIZE
        page_num = request.GET.get("page_num")
        p = Paginator(selling_products, page_size)
        cur_page = p.page(1)
        if page_num is not None:
            cur_page = p.page(page_num)

        context["num_pages"] = p.num_pages
        context["pages"] = range(1, p.num_pages+1)
        return render(request, "boec_core/customer/shop-grid.html",context)
class VariantDetailView(View):
    def get(self, request, *args, **kwargs):
        context = {}
        user = request.user
        variant_id = kwargs.get('variant_id')
        variant = ProductVariant.objects.get(pk=variant_id)
        options = VariantOption.objects.filter(product=variant)
        reviews = CustomerReview.objects.filter(product=variant)
        recommend_products = ProductVariant.objects.filter(is_selling=True, is_feature=True)[:6]
        if 'cart' not in request.session:
            cart = []
        else:
            cart = request.session['cart']
        if user.is_authenticated:
            context["user_id"] = user
            context["favorite"] = Favorite.objects.filter(customer=user).count()
        rating = 0
        for review in reviews:
            rating += review.rating
            replies = Reply.objects.filter(review=review)
            setattr(review, "replies", replies)

        if reviews.count() == 0:
            rating = 0
            context["is_float"] = False
        else:
            rating = rating/reviews.count()
            if(rating%1!=0):
                context["is_float"] = True
        context["cart"] = len(cart)
        context["variant"] = variant
        context["options"] = options
        context["reviews"] = reviews
        context["price"]   = options.first().price
        context["ratings"] = int(rating)
        context["max_rating"] = [1,2,3,4,5]
        context["current_rating"] = [1] * int(rating)
        return render(request, "boec_core/customer/shop-details.html",context)

class SuccessView(View):
    def get(self, request, *args, **kwargs):
        context = {}
        user = request.user
        return render(request, "boec_core/customer/success_page.html",context)

class FailureView(View):
    def get(self, request, *args, **kwargs):
        context = {}
        user = request.user
        return render(request, "boec_core/customer/failure_page.html",context)
class FailureVoucherView(View):
    def get(self, request, *args, **kwargs):
        context = {}
        user = request.user
        return render(request, "boec_core/customer/failure_voucher.html",context)
class CheckoutView(View):
    def post(self, request, *args, **kwargs):
        user = request.user
        cart = {}
        cart_items = []
        total = 0
        name = request.POST.get("name")
        phone = request.POST.get("phone")
        email = request.POST.get("email")
        # Lấy các thông tin được gửi qua post request
        city = int(request.POST.get("city"))
        district = int(request.POST.get("district"))
        street = int(request.POST.get("street"))
        department = int(request.POST.get("department"))
        voucher = request.POST.get("voucher")
        detail = request.POST.get("detail")
        city = City.objects.get(pk=city)
        district = District.objects.get(pk=district)
        street = Street.objects.get(pk=street)
        department = ShippingDepartment.objects.get(pk=department)
        has_voucher = voucher
        voucher = Voucher.objects.filter(code=voucher).first()
        note = request.POST.get("note")
        # hiện tại chưa có api lấY giá vận chuyểN -> mặc định giá vậN chuyển là 30
        shipping_fee = 30
        # nếu user đăng nhập
        if voucher is None and has_voucher is not None:
            return HttpResponseRedirect("/boec/failure_voucher")
        if user.is_authenticated:
            order = Order(recv_name=name,recv_city=city, recv_district = district, recv_street=street, recv_house=detail, recv_phone=phone,
            recv_email=email,note=note,customer=user,payment_type="COD")
        else:
            order = Order(recv_name=name,recv_city=city, recv_district = district, recv_street=street, recv_house=detail,recv_phone=phone,
            recv_email=email,note=note,payment_type="COD")
        try:
            order.save()
            if 'cart' not in request.session:
                cart = {}
            else:
                cart = request.session['cart']
            for item in cart:
                variant = VariantOption.objects.get(pk=item)
                quantity = cart[item]
                amount = quantity * variant.price
                ordered_product = OrderedProduct(quantity=quantity, price=variant.price,
                order=order, product=variant)
                variant.quantity -= quantity
                total+=amount
                variant.save()
                ordered_product.save()
            if voucher:
                if voucher.role == 1:
                    shipping_fee = shipping_fee - voucher.offset
                elif voucher.role == 2:
                    total = total - (total * voucher.sale)/100
                order.voucher = voucher
            # chưa có api của bên giao hàng nên bọn em sẽ tự đạt ra ngày nhận hàng là 3 ngày sau
            estimated = date.today()+timedelta(days=3)

            # Tạo thông tin ship hàng
            ordership = OrderShip(estimated_arrival=estimated, price = shipping_fee, ship=department, order=order)
            ordership.save()
            # Cập nhật đơn hàng
            order.amount = total + shipping_fee
            order.ship = shipping_fee
            order.save()
            # Xoá giỏ hàng cũ
            del request.session['cart']
            return HttpResponseRedirect("/boec/success")
        except:
            return HttpResponseRedirect("/boec/failure")

    def get(self, request, *args, **kwargs):
        context = {}
        user = request.user
        context["user"] = user
        cart = {}
        cart_items = []
        total = 0
        cities = City.objects.all()
        districts = District.objects.all()
        streets = Street.objects.all()
        departments = ShippingDepartment.objects.all()
        address = ShippingAdress.objects.filter(customer=user)
        if 'cart' not in request.session:
            cart = {}
        else:
            cart = request.session['cart']
        for item in cart:
            variant = VariantOption.objects.get(pk=item)
            quantity = cart[item]
            amount = quantity * variant.price
            setattr(variant, 'quantity', quantity)
            setattr(variant, 'total', amount)
            total += amount
            cart_items.append(variant)
        estimated = date.today()+timedelta(days=3)
        context["cart"] = len(cart)
        context["cart_items"] = cart_items
        context["cities"] = cities
        context["districts"] = districts
        context["streets"] = streets
        context["address"] = address
        context["total"] = total
        context["departments"] = departments
        context["estimated"] = estimated

        return render(request, "boec_core/customer/checkout.html",context)


import pygame
import pygame.freetype
import random

#Variables and constants
FPS =  75
UNIT_SIZE = 25
SCREEN_WIDTH = 500
SCREEN_HEIGHT = 500

xDirection = 1
yDirection = 0

foodX = 0
foodY = 0

score = 0
scoreText = "0"

run = False

#initial snake values
bodyParts = 5
snake = [
    {"x": UNIT_SIZE*4, "y": 0}, #head index 0
    {"x": UNIT_SIZE*3, "y": 0},
    {"x": UNIT_SIZE*2, "y": 0},
    {"x": UNIT_SIZE*1, "y": 0},
    {"x": UNIT_SIZE*0, "y": 0}  #tail index 5
]

pygame.init()

SCORE_FONT = pygame.freetype.SysFont('Comic Sans MS', 30)
GAME_OVER_FONT = pygame.freetype.SysFont('Comic Sans MS', 70)
RESET_FONT = pygame.freetype.SysFont('Comic Sans MS', 20)

window = pygame.display.set_mode((SCREEN_WIDTH, SCREEN_HEIGHT))
pygame.display.set_caption("Snake in Python")

#methods
def startGame():
    global xDirection
    global yDirection

    xDirection = 1
    yDirection = 0

    newFood()
    gameLoop()

def newFood():

    global foodX
    global foodY

    foodY = random.randint(0, SCREEN_HEIGHT / UNIT_SIZE) * UNIT_SIZE
    foodX = random.randint(0, SCREEN_WIDTH / UNIT_SIZE) * UNIT_SIZE

    #Prevents food from spawning offscreen
    if foodX >= SCREEN_WIDTH:
        foodX -= UNIT_SIZE
    if foodY >= SCREEN_HEIGHT:
        foodY -= UNIT_SIZE

def drawFood():
    color = (255, 0, 0)
    pygame.draw.rect(window, color, pygame.Rect(foodX, foodY, UNIT_SIZE, UNIT_SIZE))

def drawSnake():
    for i in range(bodyParts):
        color = (0, 255, 0)
        pygame.draw.rect(window, color, pygame.Rect(snake[i]["x"], snake[i]["y"], UNIT_SIZE, UNIT_SIZE))

def moveSnake(xDir, yDir):

    global bodyParts
    global score
    global scoreText

    #moving the head of the snake
    snake.insert(0, {"x": (snake[0]["x"] + (xDir * UNIT_SIZE)), "y": (snake[0]["y"] + (yDir * UNIT_SIZE))})

    #if the snake eats food
    if snake[0]["x"] == foodX and snake[0]["y"] == foodY:
        score += 1
        scoreText = str(score)
        bodyParts += 1
        newFood()

    for i in range(bodyParts):
        if i == bodyParts:
            snake.pop(i)

def getDirection(key):
    #user input
    global xDirection
    global yDirection

    if key == pygame.K_w or key == pygame.K_UP:
        if yDirection != 1:
            yDirection = -1
            xDirection = 0
    elif key == pygame.K_s or key == pygame.K_DOWN:
        if yDirection != -1:
            yDirection = 1
            xDirection = 0
    elif key == pygame.K_a or key == pygame.K_LEFT:
        if xDirection != 1:
            xDirection = -1
            yDirection = 0
    elif key == pygame.K_d or key == pygame.K_RIGHT:
        if xDirection != -1:
            xDirection = 1
            yDirection = 0

def checkGameOver():

    if snake[0]["x"] >= SCREEN_WIDTH or snake[0]["x"] < 0 or snake[0]["y"] >= SCREEN_HEIGHT or snake[0]["y"] < 0:
        return False

    for i in range(bodyParts):
        if i != 0:
            if snake[0]["x"] == snake[i]["x"] and snake[0]["y"] == snake[i]["y"]:
                return False
    #if no game over occurs
    return True

def gameOver():

    global xDirection, yDirection, score, scoreText, snake, bodyParts

    window.fill((0,0,0))

    SCORE_FONT.render_to(window, (SCREEN_WIDTH/2 - 80, 10), "Final Score: "+scoreText, (0, 150, 255))
    GAME_OVER_FONT.render_to(window, (50, SCREEN_HEIGHT/2 - 20), "GAME OVER!", (255, 0, 50))
    RESET_FONT.render_to(window, (130, SCREEN_HEIGHT/2 + 75), "Press any key to restart", (0, 150, 255))

    pygame.display.update()

    resetCheck = True
    while resetCheck:
        for event in pygame.event.get():

            if event.type == pygame.QUIT:
                resetCheck = False
                break
            elif event.type == pygame.KEYDOWN:

                #resetting to default values

                xDirection = 1
                yDirection = 0

                score = 0
                scoreText = "0"

                bodyParts = 5

                snake = [
                    {"x": UNIT_SIZE*4, "y": 0}, #head index 0
                    {"x": UNIT_SIZE*3, "y": 0},
                    {"x": UNIT_SIZE*2, "y": 0},
                    {"x": UNIT_SIZE*1, "y": 0},
                    {"x": UNIT_SIZE*0, "y": 0}  #tail index 5
                ]

                startGame()
                resetCheck = False

def gameLoop():

    global run

    #Game loop
    run = True
    while run:
        #resets window
        window.fill((0,0,0))

        #looks through events
        for event in pygame.event.get():
            #quits game
            if event.type == pygame.QUIT:
                run = False

            if event.type == pygame.KEYDOWN:
                getDirection(event.key)

        if checkGameOver() == False:
            run = False
            gameOver()

        moveSnake(xDirection, yDirection)
        drawSnake()
        drawFood()

        SCORE_FONT.render_to(window, (SCREEN_WIDTH/2 - 60, 10), "Score: "+scoreText, (0, 150, 255))

        pygame.display.update()

        pygame.time.wait(int(FPS))

#testing
startGame()

pygame.quit()


# ----------------------------------------------------------------------------------------------------------------------
#                                                     Imports
# ----------------------------------------------------------------------------------------------------------------------
import torch
import torch.nn as nn
import torch.nn.functional as tf
from .PytorchNet import PytorchNet
from util.gen import banner
from math import log10,floor,ceil

# ----------------------------------------------------------------------------------------------------------------------
#                                              Spatial Layer Implementation
# ----------------------------------------------------------------------------------------------------------------------
class Spatial(nn.Module):
    def __init__(self, channels,conv_to_MAC):
        super().__init__()

        self.channels = channels
        self.conv_to_MAC = conv_to_MAC
        self.ops_saved = 0
        self.total_ops = 0
        self.total_conv = 0
        self.conv_saved = 0
        self.is_init = False
        self.enable = False
        self.mask = None

        self.p_size, self.batch_size, self.in_shape, = (None, None, None)
        self.padded_in_shape, self.use_cuda, self.pad_s = (None, None, None)
        self.conv_filt, self.batch_mask = (None, None)

    def init_to_input(self, p_size, batch_size, in_shape, padded_in_shape, use_cuda):

        self.p_size, self.batch_size, self.in_shape, self.padded_in_shape, self.use_cuda = \
            p_size, batch_size, in_shape, padded_in_shape, use_cuda

        self.pad_s = self.padded_in_shape[1] - self.in_shape[1]
        self.conv_filt = nn.Conv2d(self.channels, self.channels, kernel_size=self.p_size, stride=self.p_size,
                                   bias=False, groups=self.channels)
        self.conv_filt.weight.data.fill_(1)
        # Make convolution later constant on backward passes
        for p in self.conv_filt.parameters():
            p.requires_grad = False

        if use_cuda:
            self.conv_filt = self.conv_filt.cuda()

        self.is_init = True

    def set_constant_mask(self, val):
        # To be used only after layer is initialized to input size
        assert self.is_init  # Did not include this in set_mask to savee complexity
        self.set_mask(val * torch.ones(self.padded_in_shape))

    def set_mask(self, mask):
        # To be used only after layer is initialized to input size
        self.mask = mask
        self.batch_mask = self.mask.unsqueeze(0).repeat(self.batch_size, 1, 1, 1)
        if self.use_cuda:
            self.mask = self.mask.cuda()  # TODO - Check if we can already allocate on GPU, maybe with set_
            self.batch_mask = self.batch_mask.cuda()

    def set_enable(self, enable):
        # To be used only after layer is initialized to input size
        self.enable = enable

    def num_ops_saved(self,use_conv=False):
        if use_conv:
            return self.conv_saved
        else:
            return self.ops_saved

    def num_ops_total(self,use_conv=False):
        if use_conv:
            return self.total_conv
        else:
            return self.total_ops

    def reset_ops(self):
        self.total_ops = 0
        self.ops_saved = 0
        self.conv_saved =0
        self.total_conv = 0

    def forward(self, x):
        if not self.enable:
            # print('Layer')
            return x

        if self.pad_s != 0:
            x = torch.nn.functional.pad(x, (0, self.pad_s, 0, self.pad_s), value=0)  # Pad with ZEROS

        if x.size(0) != self.batch_size:
            #print('Batch size event')  # - Will happen once every test forward
            batch_mask = self.batch_mask[:x.size(0), :, :, :]
        else:
            batch_mask = self.batch_mask

        # The convolution basically sums over all non-zero cells. We get a block predicator for each patch
        # Anywhere that is zero - That's where we are saving operations
        b = (self.conv_filt(torch.mul(x, batch_mask)) > 0).float()
        # TODO - Check if this could be shortened (Maybe via the Pytorch Upsample module?)
        b_expanded = b.repeat(1, 1, self.p_size, self.p_size). \
            reshape(b.size(0), b.size(1), self.p_size, -1).permute(0, 1, 3, 2). \
            reshape(b.size(0), b.size(1), b.size(2) * self.p_size, -1)

        self.conv_saved += torch.sum(torch.mul(1 - b_expanded, 1 - batch_mask))
        self.total_conv += x.shape[0] * x.shape[1] * x.shape[2] * x.shape[3]
        self.ops_saved += self.conv_to_MAC*self.conv_saved
        self.total_ops += self.conv_to_MAC*self.total_conv
        # Out predicator (after padding removal)
        if self.pad_s == 0:
            return torch.mul(x, b_expanded)
        else:
            return torch.mul(x, b_expanded)[:, :, :self.in_shape[1], :self.in_shape[1]]

# ----------------------------------------------------------------------------------------------------------------------
#                                        Spatial Net Abstract Class
# ----------------------------------------------------------------------------------------------------------------------
class SpatialNet(PytorchNet):
    def __init__(self, device):
        super().__init__(device)

        self.spatial_layers = []
        self.sp = None  # Will be set on first init of spatial layers - is a tuple
        self.sp_padded = None  # Will be set on first init of spatial layers - is a tuple
        self.x_shape = None
        self.p_size = None
        self.fam_name = None
        self.clustering_indices = None # Used for clustering layers together

    def forward(self, x):
        # Make it abstract
        raise NotImplementedError

    def family_name(self):
        # Abstract
        assert self.fam_name is not None # Needs to be calculated
        return self.fam_name

    def reset_spatial(self):
        for sp in self.spatial_layers:
            sp.reset_ops()
            sp.set_enable(False)

    def reset_ops(self):
        for sp in self.spatial_layers:
            sp.reset_ops()

    def name(self):
        return self.__class__.__name__

    def num_ops(self):
        ops_saved = 0
        total_ops = 0
        for sp in self.spatial_layers:
            ops_saved += int(sp.ops_saved)
            total_ops += sp.total_ops
        return ops_saved, total_ops

    def num_conv(self):
        conv_saved = 0
        total_conv = 0
        for sp in self.spatial_layers:
            conv_saved += int(sp.conv_saved)
            total_conv += sp.total_conv
        return conv_saved, total_conv

    def print_ops_summary(self,use_conv=False):

        if use_conv:
            name = 'Conv'
            all_ops_saved, all_total_ops = self.num_conv()
        else:
            name = 'MAC'
            all_ops_saved, all_total_ops = self.num_ops()

        for i, l in enumerate(self.spatial_layers):

            total_ops = l.num_ops_total(use_conv)
            saved_ops = l.num_ops_saved(use_conv)

            if total_ops == 0:
                print(f'Spatial Layer {i}: {name} Ops saved: {saved_ops}/{total_ops}')
            else:
                spacer = ' ' * (len(str(len(self.spatial_layers))) - len(str(i)))
                assert(saved_ops <= all_ops_saved)
                if all_ops_saved ==0: # Handle div by 0 case
                    midstr = '0'
                else:
                    midstr = f'{l.ops_saved*100/all_ops_saved:.3f}'
                print(
                    f'Spatial Layer {i}:{spacer} {name} Ops saved: {saved_ops*100 /total_ops:.3f} % [{int(saved_ops)} / {total_ops}]',
                    f'of [{midstr} % / {total_ops*100/all_total_ops:.3f} %]')

        if all_total_ops > 0:
            print(f'Grand total [{name}]: {all_ops_saved}/{all_total_ops} {all_ops_saved*100/all_total_ops:.3f} %')
        else:
            print(f'Grand total [{name}]: {all_ops_saved}/{all_total_ops}')

    def initialize_spatial_layers(self, x_shape, batch_size, p_size,freeze=True):

        if freeze:
            self.eval() #LOCK Network for testing only
        # From init phase and on, set the spatial sizes
        self.x_shape = x_shape
        self.p_size = p_size
        self.sp = self.generate_spatial_sizes(x_shape)
        self.sp_padded = self.generate_padded_spatial_sizes(x_shape, p_size)

        for i, layer in enumerate(self.spatial_layers):
            layer.init_to_input(p_size=p_size, batch_size=batch_size, in_shape=self.sp[i],
                                padded_in_shape=self.sp_padded[i],
                                use_cuda=self.use_cuda)

    def num_spatial_layers(self):
        return len(self.spatial_layers)

    def enabled_layers(self):
        return [i for i in range(self.num_spatial_layers()) if self.spatial_layers[i].enable]

    def disabled_layers(self):
        return [i for i in range(self.num_spatial_layers()) if not self.spatial_layers[i].enable]

    def enable_spatial_layers(self, idx_list):
        for resurrected in idx_list:
            self.spatial_layers[resurrected].set_enable(True)

    def disable_spatial_layers(self, idx_list):
        for goner_id in idx_list:
            self.spatial_layers[goner_id].set_enable(False)

    def print_spatial_status(self):
        init_status = ['-Initialized-' if sp.is_init else '-Uninitialized-' for sp in self.spatial_layers]
        enable_status = ['-Enabled-' if sp.enable else '-Disabled-' for sp in self.spatial_layers]
        mask_status = ['-Mask Not Set-' if sp.mask is None else '-Mask Set-' for sp in self.spatial_layers]
        banner('Spatial Status')
        for i, (iS, eS, mS) in enumerate(zip(init_status, enable_status, mask_status)):
            print(f'Spatial Layer {i}: {iS} {eS} {mS}')

    def strict_mask_update(self, update_ids, masks):
        # Turn on all the update ids
        self.lazy_mask_update(update_ids, masks)

        # Turn off all others
        disabled = [i for i in range(len(self.spatial_layers)) if i not in update_ids]
        self.disable_spatial_layers(disabled)

    def lazy_mask_update(self, update_ids, masks):
        for (i, mask) in zip(update_ids, masks):
            self.spatial_layers[i].set_mask(mask)
            self.spatial_layers[i].set_enable(True)

    def fill_masks_to_val(self, val):
        for layer in self.spatial_layers:
            layer.set_constant_mask(val)
            layer.set_enable(True)

    def generate_spatial_sizes(self, x_shape):
        # NOTE - This returns the sizes *WITHOUT* the padding and wit
        if self.sp is None or self.x_shape != tuple(x_shape):  # Spatial layers were not init or new dataset
            summary = self.summary(x_shape, print_it=False)
            sp = tuple(tuple(value['input_shape'][1:]) for key, value in summary.items() if key.startswith('Spatial'))

            if self.clustering_indices is not None:
                sp = tuple([sp[i] for i in self.clustering_indices])
            return sp
        else:
            return self.sp

    def generate_padded_spatial_sizes(self, x_shape, p_size):

        if self.sp_padded is None or self.p_size != p_size or self.x_shape != x_shape:
            sp_padded = []
            sps = self.generate_spatial_sizes(x_shape)
            for param in sps:
                pad_s = param[1] % p_size
                if pad_s != 0:
                    pad_s = p_size - pad_s
                    sp_padded.append((param[0], param[1] + pad_s, param[2] + pad_s))
                else:
                    sp_padded.append(param)
            return tuple(sp_padded)
        else:
            return self.sp_padded

    # Override
    def summary(self, x_shape, batch_size=-1, print_it=True):
        # TODO - Make this "careful" activation wrapper a built in private function
        if self.sp is None:
            return super().summary(x_shape, print_it=print_it)
        else:
            enabled = self.enabled_layers()
            if not enabled:
                return super().summary(x_shape, print_it=print_it)
            else:
                self.disable_spatial_layers(enabled)
                summary = super().summary(x_shape, print_it=print_it)
                self.enable_spatial_layers(enabled)
                return summary

    def output_size(self, x_shape,cuda_allowed=False):

        if self.sp is None:
            return super().output_size(x_shape,cuda_allowed)
        else:
            enabled = self.enabled_layers()
            if not enabled:
                return super().output_size(x_shape,cuda_allowed)
            else:
                self.disable_spatial_layers(enabled)
                siz = super().output_size(x_shape,cuda_allowed)
                self.enable_spatial_layers(enabled)
                return siz

def sequential_spatial_layer_extract(seq_model):
    spatial_layers = []
    for idx, m in enumerate(seq_model.modules()):
        #print(idx, '->', m)
        if isinstance(m,Spatial):
            spatial_layers.append(m)
    return spatial_layers


import json
from datetime import datetime
import sys
import os
import re

import cloudflare
import config
import webp
import wp
from loggr import logger

db_prefix, db_prefix_err = wp.DBPrefix().run()
app_path = config.get().get("APP_PATH").rstrip("/")

# ENUMs for actions taken.
CONVERTED = "converted"
PROCESSED = "processed"
NOT_PROCESSED = "not_processed"

# Gets the backup filename.
def get_backup_filename(date=""):
    """
    Gets the backup filename.
    :param date: str, optional
    :return: str
    """
    if "" == date:
        date = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")

    cfg = config.get()
    path = cfg.get("BACKUPS_PATH")
    if not os.path.exists(path):
        os.makedirs(path)

    domain = cfg.get("SITE_DOMAIN")
    if domain is None:
        domain = 'wordpress'
    return os.path.join(path, f'backup-{domain}-{date}.sql')

# Import DB
def import_db(date):
    """
    Restores the DB using WP CLI `wp db export`.
    :return: tuple of output
    """
    db_import = wp.DBImport(
        get_backup_filename(date),
        skip_optimizations=True,
        as_sudo=(os.geteuid() == 0),
    )
    return db_import.run()

# Backup DB
def export_db():
    """
    Backups the DB using WP CLI `wp db export`.
    :return: tuple of output
    """
    export = wp.DBExport(
        get_backup_filename(),
        add_drop_table=True,
        as_sudo=(os.geteuid() == 0),
    )
    return export.run()

# Gets the hostname/site domain from config.
def get_domain():
    """
    Gets the hostname/site domain from config.
    :return: str
    """
    return config.get().get("SITE_DOMAIN")

# Gets the app path from config.
def get_path():
    """
    Gets the app path from config using the global `app_path`.
    :return: str
    """
    return app_path

# Gets the URL pattern for a domain (without http/s) to an image.
def get_domain_pattern():
    """
    Gets the URL pattern for a domain (without http/s) to an image.
    :return: str
    """
    return f'({get_domain()})([-a-zA-Z0-9()@:%_\+.~#?&\/=]+?([\w\d_-]+)\.)(jpg|jpeg|png|gif)'

# Gets the path pattern for a domain to an image.
def get_path_pattern():
    """
    Gets the path pattern for a domain to an image.
    :return: str
    """
    return f'({get_path()})([-a-zA-Z0-9()@:%_\+.~#?&\/=]+?([\w\d_-]+)\.)(jpg|jpeg|png|gif)'

# Get DB Prefix.
def get_db_prefix():
    """
    Gets DB Prefix.
    :return: str
    """
    return db_prefix

# Purges the WP and Cloudflare cache.
def purge_cache():
    """
    Purges the WP and Cloudflare cache.
    :return: bool whether cache was cleared or not.
    """
    was_flushed = False

    if config.get().get("FLUSH_DATABASE"):
        out, err = wp.CacheFlush(
            as_sudo=(os.geteuid() == 0),
        ).run()
        if "Success: The cache was flushed." != out.strip():
            was_flushed = True

    if config.get().get("CF_PURGE_CACHE"):
        is_flushed = cloudflare.purge(config.get().get("CF_ZONE_ID"))
        if is_flushed:
            was_flushed = True

    return was_flushed

# Prepares the DB for this script to run and test.
def prepare():
    """
    Creates a backup of the DB for this script to run.
    :return: void
    """
    out, err = export_db()
    logger.info(out)
    logger.debug(err)

# Get image paths from search output for a specific table.
def get_image_paths_from_table_out(out):
    """
    Get image paths from search output for a specific table.
    :param out: String of wp db search output for a specific table.
    :return: list of image paths.
    """
    image_paths = []
    domain_matches = re.findall(get_domain_pattern(), out)
    for match in domain_matches:
        image_paths.append(os.path.join(get_path(), match[1].lstrip("/")) + match[3])

    path_matches = re.findall(get_path_pattern(), out)
    for match in path_matches:
        image_paths.append(os.path.join(get_path(), match[1].lstrip("/")) + match[3])

    return list(set(image_paths))

# Get image paths by table.
def get_image_paths_by_table(out):
    """
    Get image paths by table.
    :param out: String of wp db search output.
    :return: Tables object.
        {
            "table": "", Table name
            "out": "",   wp db search output for that table
            "ids": [],   IDs of the DB rows
        }
    """
    tables = {}
    matches = re.findall(r'(\w+):(\w+):(\d+):(.*)', out)

    for match in matches:
        if match[0] not in tables:
            tables[match[0]] = {"table": match[0], "out": '', "images": [], "ids": {}}
        tables[match[0]]["out"] += "\n".join(match)

        if match[1] not in tables[match[0]]["ids"]:
            tables[match[0]]["ids"][match[1]] = []
        tables[match[0]]["ids"][match[1]].append(match[2])

    for key, table in tables.items():
        tables[key]["images"] = get_image_paths_from_table_out(table["out"])

    return tables

# Process images, starting with the database.
def process_db_images():
    """
    Process images, starting with the database.
    :return: Tuple of images processed, not processed, converted
    """
    db_images_processed, db_images_not_processed, db_images_converted = [], [], []

    # Find all the images.
    s = wp.DBSearch(
        get_domain_pattern(),
        as_sudo=(os.geteuid() == 0),
        regex=True,
        all_tables=True,
        # table_column_once=True,
        one_line=True,
    )
    out, err = s.run()
    logger.info(out)
    logger.debug(err)

    # Process the images
    table_images = get_image_paths_by_table(out)

    # Process attachments first.
    if f"{get_db_prefix()}posts" in table_images and "guid" in table_images[f"{get_db_prefix()}posts"]["ids"]:
        table = table_images[f"{get_db_prefix()}posts"]

        # Only attachments will match in the guid...most likely.
        for index, ID in enumerate(table["ids"]["guid"]):
            # Get the Post.
            pg = wp.PostGet(ID, format="json", fields="post_type,post_mime_type,guid")
            out, err = pg.run()
            post = json.loads(out)

            # Maybe convert the image.
            o_file = re.sub(rf"https?:\/\/{get_domain()}", get_path(), post["guid"])
            w_file = get_webp_image_file(o_file)
            result = process_image(o_file)

            # Don't do anything else if the image was not processed.
            if NOT_PROCESSED == result:
                continue

            # Update webp image and mime type.
            if "attachment" == post["post_type"]:
                guid = re.sub(rf'({"|".join(get_image_exts())})', 'webp', post["guid"])

                # Update the post guid, post_mime_type.
                # Cannot update via wp_update_post (wp.PostUpdate).
                dbq = wp.DBQuery(f"UPDATE {table['table']} "
                                 f"SET guid=\"{guid}\", post_mime_type = \"image/webp\" "
                                 f"WHERE ID = {ID}")
                out, err = dbq.run()
                logger.info(f"Updating attachment ({ID})")
                logger.info(out)
                logger.debug(err)

                # Maybe regenerate the attachment metadata.
                # Currently there is no WP CLI command to do this sadly.
                # I probably should write one for them.
                e = wp.Eval(f"$data = wp_generate_attachment_metadata({ID}, '{w_file}');" +
                            f"if ( wp_update_attachment_metadata({ID}, $data) && update_attached_file( {ID}, '{w_file}' ) ) " + "{"
                                                                                "  echo 'true';"
                                                                                "} else { "
                                                                                f"  throw Exception('Could not update attachment metadata for {ID}'); "
                                                                                "}")
                out, err = e.run()
                logger.info(out)
                logger.debug(err)

    # for table in table_images:
    for key, table in table_images.items():

        # Now process the remaining images.
        images_processed, images_not_processed, images_converted = process_images(table["images"])
        db_images_processed += images_processed
        db_images_not_processed += images_not_processed
        db_images_converted += images_converted

        # Do all processed images.
        for o_file in images_processed:
            sr = wp.SearchReplace(
                o_file.replace(get_path(), ""),
                get_webp_image_file(o_file).replace(get_path(), ""),
                table=table["table"],
                as_sudo=(os.geteuid() == 0),
                skip_themes=True,
                skip_plugins=True,
                include_columns=",".join(table["ids"].keys()),
                all_tables=True,
                color=False)
            out, err = sr.run()
            logger.info(sr)
            logger.info(out)
            logger.debug(err)

    return db_images_processed, db_images_not_processed, db_images_converted

# Gets the supported image file extensions.
def get_image_exts():
    """
    Gets the supported image file extensions.
    :return: List of extensions
    """
    return ["jpg", "jpeg", "png", "gif"]

# Processes a directory.
def process_dir(file_dir):
    """
    Processes a directory.
    :param file_dir: File directory string
    :return: Tuple of images processed, not processed, converted
    """
    dir_images_processed, dir_images_not_processed, dir_images_converted = [], [], []
    if os.path.exists(file_dir) and os.path.isdir(file_dir):
        logger.info(f"Processing DIR {file_dir}")
        file_dir = file_dir.rstrip("/") + "/"
        files = os.listdir(file_dir)
        files = [os.path.join(file_dir, f) for f in files]
        dirs = [f for f in files if os.path.isdir(f)]
        files = [f for f in files if os.path.isfile(f) and f.split('.')[-1:][0] in get_image_exts()]

        if len(files) > 0:
            images_processed, images_not_processed, images_converted = process_images(files)
            dir_images_processed += images_processed
            dir_images_not_processed += images_not_processed
            dir_images_converted += images_converted

        for d in dirs:
            images_processed, images_not_processed, images_converted = process_dir(d)
            dir_images_processed += images_processed
            dir_images_not_processed += images_not_processed
            dir_images_converted += images_converted
    else:
        msg = f"'{file_dir}' either doesn't exist, or is not a dir..."
        logger.error(msg)
        print(msg)

    return dir_images_processed, dir_images_not_processed, dir_images_converted

# Prints the summary.
def print_summary(action, images):
    """
    Prints a summary.
    :param action: ENUM of actions.
    :param images: List of image filenames
    :return: void
    """
    logger.info(f"---- {action.upper()} ----\n{images}\n---------------------------------")

# Gets the webp image file from the original file.
def get_webp_image_file(o_file):
    """
    Gets the webp image file from the original file.
    :param o_file: Filepath string.
    :return: webp filepath string.
    """
    ext = o_file.split('.')[-1:][0]
    return o_file.replace(f".{ext}", '.webp')

# Processes a single image file, converting to webp.
def process_image(o_file):
    """
    Processes a single image file, converting to webp.
    :param o_file: Filepath string.
    :return: ENUM of processing result.
    """
    if o_file.endswith(tuple(get_image_exts())):
        w_file = get_webp_image_file(o_file)
        if os.path.exists(o_file):
            if not os.path.exists(w_file):
                webp.convert(o_file, w_file)
                logger.info(f"'{o_file}' Converted to '{w_file}'")
                return CONVERTED

            logger.debug(f"'{w_file}' exists")
            return PROCESSED

        logger.error(f"Original '{o_file}' does not exist")
        return NOT_PROCESSED

    logger.error(f"'{o_file}' not an image")
    return NOT_PROCESSED

# Processes a set of image files, converting them to webps.
def process_images(image_paths):
    """
    Processes a set of image files, converting them to webps.
    :param image_paths: List of image paths.
    :return: Tuple of string of filenames.
    """
    images_processed = []
    images_converted = []
    images_not_processed = []
    for o_file in image_paths:
        logger.debug(f"  -Processing FILE {o_file}")
        action = process_image(o_file)
        if "converted" == action:
            images_converted.append(o_file)

        if "converted" == action or "processed" == action:
            images_processed.append(o_file)

        if "not_processed" == action:
            images_not_processed.append(o_file)

    return images_processed, images_not_processed, images_converted

# RUNS everything.
def run():
    """
    Runs everything.
    :return: void
    """
    fs_images_processed, fs_images_not_processed, fs_images_converted = process_dir(get_path())
    print_summary("filesystem images processed", fs_images_processed)
    print_summary("filesystem images converted", fs_images_converted)
    print_summary("filesystem images not processed", fs_images_not_processed)

    db_images_processed, db_images_not_processed, db_images_converted = process_db_images()
    print_summary("DB images - processed", db_images_processed)
    print_summary("DB images - converted", db_images_converted)
    print_summary("DB images - not processed", db_images_not_processed)

    if len(fs_images_converted or db_images_converted) > 0:
        purge_cache()

if __name__ == '__main__':
    if len(sys.argv) > 1:
        app_path = sys.argv[1]
    prepare()
    run()


import matplotlib
matplotlib.use("TkAgg")
import matplotlib.pyplot as plt

import pickle
import numpy as np
from pandas import read_csv
from pandas import DataFrame
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix

def load_data(dtree=False):
	data = read_csv("data/pulsar_stars.csv")
	y_data = np.array(data['target_class'])
	x = data.drop(['target_class'], axis=1)
	x_data = np.array([(x[column] - np.min(x[column]))/(np.max(x[column]) - np.min(x[column])) for column in x]).transpose() #scale between 0/1

	x_train, x_test, y_train, y_test = train_test_split(x_data, y_data, test_size=0.25, random_state=0)

	if dtree:
		x_test, x_prune, y_test, y_prune = train_test_split(x_test, y_test, test_size=0.5, random_state=0)

		train_test_data = {'x_train': x_train,
						   'y_train': y_train,
						   'x_test': x_test,
						   'y_test': y_test,
						   'x_prune': x_prune,
						   'y_prune': y_prune}
		with open('train_test_data_dtree.pkl', 'wb') as output:
			pickle.dump(train_test_data, output, -1)

	else:
		train_test_data = {'x_train': x_train,
						   'y_train': y_train,
						   'x_test': x_test,
						   'y_test': y_test}

		with open('train_test_data.pkl', 'wb') as output:
			pickle.dump(train_test_data, output, -1)

	print('(Training data loaded successfully)\n')

def read_data(dtree=False):
	if dtree:
		with open('train_test_data_dtree.pkl', 'rb') as input:
			train_test_data = pickle.load(input)

		x_train = train_test_data['x_train']
		y_train = train_test_data['y_train']

		x_test = train_test_data['x_test']
		y_test = train_test_data['y_test']

		x_prune = train_test_data['x_prune']
		y_prune= train_test_data['y_prune']

		print('(Training data read successfully)\n')

		return x_train, y_train, x_test, y_test, x_prune, y_prune

	else:
		with open('train_test_data.pkl', 'rb') as input:
				train_test_data = pickle.load(input)

		x_train = train_test_data['x_train']
		y_train = train_test_data['y_train']

		x_test = train_test_data['x_test']
		y_test = train_test_data['y_test']

		print('(Training data read successfully)\n')

		return x_train, y_train, x_test, y_test

def view_data(data):

	keys = list(data)

	print(keys)

	noise_points = data[data.target_class==0]
	pulsar_points = data[data.target_class==1]

	fig, ax = plt.subplots(2, 2)

	ax[0, 1].scatter(noise_points[keys[1]], noise_points[keys[2]],label="RF Noise",color="red",alpha=0.1)
	ax[0, 1].scatter(pulsar_points[keys[1]], pulsar_points[keys[2]],label="Pulsar",color="purple",alpha=0.1)
	ax[0, 1].set_xlabel(keys[1])
	ax[0, 1].set_ylabel(keys[2])
	ax[0, 1].legend()

	ax[0, 0].scatter(noise_points[keys[0]], noise_points[keys[3]],label="RF Noise",color="red",alpha=0.1)
	ax[0, 0].scatter(pulsar_points[keys[0]], pulsar_points[keys[3]],label="Pulsar",color="purple",alpha=0.1)
	ax[0, 0].set_xlabel(keys[0])
	ax[0, 0].set_ylabel(keys[3])

	ax[1, 0].scatter(noise_points[keys[4]], noise_points[keys[7]],label="RF Noise",color="red",alpha=0.1)
	ax[1, 0].scatter(pulsar_points[keys[4]], pulsar_points[keys[7]],label="Pulsar",color="purple",alpha=0.1)
	ax[1, 0].set_xlabel(keys[4])
	ax[1, 0].set_ylabel(keys[7])

	ax[1, 1].scatter(noise_points[keys[5]], noise_points[keys[6]],label="RF Noise",color="red",alpha=0.1)
	ax[1, 1].scatter(pulsar_points[keys[5]], pulsar_points[keys[6]],label="Pulsar",color="purple",alpha=0.1)
	ax[1, 1].set_xlabel(keys[5])
	ax[1, 1].set_ylabel(keys[6])

	fig.suptitle('Pulsar Star vs RF Noise Attributes')
	plt.show()

def test_metrics(x_test, y_test, clf_obj, do_print=False):
	y_predict = clf_obj.predict(x_test)
	conf_mat = confusion_matrix(y_test, y_predict)

	true_pos = conf_mat[1][1]
	false_pos = conf_mat[0][1]
	false_neg = conf_mat[1][0]
	if (true_pos + false_pos) == 0:
		return 0

	precision = true_pos/(true_pos + false_pos)
	recall = true_pos/(true_pos + false_neg)
	f1 = 2 * (precision*recall) / (precision+recall)
	if do_print:
		print('Recall: ', str(recall))
		print('Precision: ', str(precision))
		print('F-measure: ', str(f1))
		print('Accuracy: ', str(accuracy_score(y_test, y_predict)*100))
		print('Confusion matrix:')
		print(conf_mat)
		print()
	return f1


import os
import pandas as pd
import pymysql.cursors

conn = pymysql.connect(host='localhost',
                       user='root',
                       password='',
                       db='albert',
                       charset='utf8mb4',
                       cursorclass=pymysql.cursors.DictCursor)
'''
class row:
	#ignore class status, session, start/end date (add sem/year ), description,
	#component = class_type
	#min/max units = credits
	#building room = room_num/building
	#primary instructor = first_name, last_name
	def __init__(self, catalog_num, class_section, title, classnbr, days,
		timest, sem, year, class_type, credits, consent, course_id, location,
		room_num, building, room_cap, enroll_cap, enroll_tot, waitlist_cap, waitlist_tot,
		combined, class_topic, first_name, last_name):
		self.catalog_num = catalog_num
		self.class_section = class_section
		self.title = title
		self.classnbr = classnbr
		self.days = days
		self.timest = timest
		self.sem = sem
		self.year = year
		self.class_type = class_type
		self.credits = credits
		self.consent = consent
		self.course_id = course_id
		self.location = location
		self.room_num = room_num
		self.building = building
		self.room_cap = room_cap
		self.enroll_cap = enroll_cap
		self.enroll_tot = enroll_tot
		self.waitlist_cap = waitlist_cap
		self.waitlist_tot = waitlist_tot
		self.combined = combined
		self.class_topic = class_topic
		self.first_name = first_name
		self.last_name = last_name

def readFile():
    rows = []
    excel_file = "data.xlsx"
    data = pd.read_excel(excel_file)
    j = 2;
    for index, row in data.iterrows():
      print("row" + str(j))
      #add sem/year
      if (row["Start/End Date"][1] == "9"):
          sem = "Fall"
      else:
          sem = "Spring"

      year = row["Start/End Date"][-4:]

      #add building

      #if empty
      if (pd.isnull(row["Building Room"])):
          building = ""
          room_num = ""
      #if online
      elif(row["Building Room"] == "On-Line"):
          building = "Online"
          room_num = "N/A"
      else:
          build = row["Building Room"].split(" Rm ")
          building = build[0]
          room_num = build[1]

      #add Instructors

      #No instructor
      if(pd.isnull(row["Primary Instructor"])):
          first_name = ""
          last_name = ""
          x = addRow(row, sem, year, room_num, building, first_name, last_name)
          rows.append(x)
     #2 instructors
      elif ("&" in row["Primary Instructor"]):
          names = row["Primary Instructor"].split("& ")
          for i in names:
              name = i.split(",")
              x = addRow(row, sem, year, room_num, building, name[0], name[1])
              rows.append(x)
     #1 instructor
      else:
          name = row["Primary Instructor"].split(",")
          x = addRow(row, sem, year, room_num, building, name[0], name[1])
          rows.append(x)
      j+=1

    course_sheet(rows)
def addRow(line, sem, year, room_num, building, first_name, last_name):
       x = row(line["Subject/Catalog#"], line["Class Section"], line["Class Title"],
                line["Class Nbr"], line["Days"], line["Time"], sem, year ,line["Component"],
                line["Min/Max Units"], line["Consent"], line["Course ID"], line["Location"],
                room_num, building, line["Room Cap"], line["Enrollment Capacity"],
                line["Enrollment Total"], line["Class Wait List Cap"], line["Wait List Total"],
                line["Combined Status"], line["Class Topic"], first_name, last_name)
       return x
'''

def course_sheet(data):

    cursor = conn.cursor()

    df = pd.DataFrame({"Course ID":data["Course ID"], "Class Title":data["Class Title"],
                  "Subject/Catalog#":data["Subject/Catalog#"],
                  "Credits":data["Min/Max Units"], "Consent":data["Consent"], "Location":data["Location"]})
    df = df.drop_duplicates("Course ID")

    for index, row in df.iterrows():
        if(row["Consent"] != "Dept"):
            df.set_value(index, "Consent", "No")

    for index, row in df.iterrows():
        query = 'INSERT INTO course(course_id, title, catalog_num, credits, consent, location) VALUES (%s, %s, %s, %s,%s,%s) ON DUPLICATE KEY UPDATE title = VALUES(title), catalog_num = VALUES(catalog_num), credits = VALUES(credits), consent = VALUES(consent), location = VALUES(location)'
        cursor.execute(query, (row["Course ID"], row["Class Title"], row["Subject/Catalog#"], row["Credits"], row["Consent"], row["Location"]))

    conn.commit()
    cursor.close()

def split_classroom(data):
    room_num = []
    building = []
    for index, row in data.iterrows():
      if (pd.isnull(row["Building Room"])):
          building.append("N/A")
          room_num.append("N/A")
      #if online
      elif(row["Building Room"] == "On-Line"):
          building.append("Online")
          room_num.append("N/A")
      elif(row["Building Room"] == "No Room Required"):
          building.append("No Room")
          room_num.append("N/A")
      elif(row["Building Room"] == "Dibner , Pfizer Auditorium"):
          building.append("Dibner")
          room_num.append("Pfizer Auditorium")
      elif("Rm" in row["Building Room"]):
          build = row["Building Room"].split(" Rm ")
          room_num.append(build[1])
          if("," in build[0]):
              building.append(build[0].rstrip(","))
          else:
              building.append(build[0])
      else:
          building.append("Unknown")
          room_num.append("Unknown")

    return room_num, building

def split_instructor(data):
    first_name = []
    last_name = []

    #add Instructors
    #No instructor
    for index, row in data.iterrows():
        if(pd.isnull(row["Primary Instructor"])):
            first_name.append("N/A")
            last_name.append("N/A")
        #2 instructors
        elif ("&" in row["Primary Instructor"]):
            names = row["Primary Instructor"].split(" & ")
            for i in names:
                name = i.split(",")
                if(len(name) > 1):
                    first_name.append(name[1])
                else:
                    first_name.append("N/A")
                if(name[0]):
                    last_name.append(name[0])
                else:
                    last_name.append("N/A")
         #1 instructor
        else:
            name = row["Primary Instructor"].split(",")
            first_name.append(name[1])
            last_name.append(name[0])
    return first_name, last_name
def classroom_sheet(data):
    #go through the data frame and add it in two separate dictionaries, add that in with room Cap
    cursor = conn.cursor()

    room_num, building = split_classroom(data)

    df = pd.DataFrame({"room_num":room_num, "building":building, "room_cap":data["Room Cap"]})
    df = df.drop_duplicates(subset=["room_num", "building"])

    df["room_cap"] = df["room_cap"].astype(str)
    for index, row in df.iterrows():
        if(row["room_cap"] == "nan"):
          df.set_value(index, "room_cap", "N/A")

    for index, row in df.iterrows():
        query = 'INSERT INTO classroom(room_num, building, room_cap) VALUES (%s, %s, %s) ON DUPLICATE KEY UPDATE room_num = VALUES(room_num), building = VALUES(building), room_cap = VALUES(room_cap)'
        cursor.execute(query, (row["room_num"], row["building"], row["room_cap"]))

    conn.commit()
    cursor.close()

def instructor_sheet(data):
    #go through data frame, parse instructor and add dictionaries and write it back in
    cursor = conn.cursor()

    first_name, last_name = split_instructor(data)

    df = pd.DataFrame({"first_name":first_name, "last_name":last_name})
    df = df.drop_duplicates(subset=["first_name", "last_name"])

    for index, row in df.iterrows():
        query = 'INSERT INTO instructor(first_name, last_name) VALUES (%s, %s) ON DUPLICATE KEY UPDATE first_name = VALUES(first_name), last_name = VALUES(last_name)'
        cursor.execute(query, (row["first_name"], row["last_name"]))

    conn.commit()
    cursor.close()

def time_sheet(data):

    cursor = conn.cursor()

    df = pd.DataFrame({"days":data["Days"], "timest":data["Time"]})
    df = df.drop_duplicates(subset=["days","timest"])

    for index, row in df.iterrows():
        if(pd.isnull(row["days"])):
            df.set_value(index, "days", "N/A")
        if(pd.isnull(row["timest"])):
            df.set_value(index, "timest", "N/A")

    for index, row in df.iterrows():
        query = 'INSERT INTO class_time(days, timest) VALUES (%s, %s) ON DUPLICATE KEY UPDATE days = VALUES(days), timest = VALUES(timest)'
        cursor.execute(query, (row["days"], row["timest"]))

    conn.commit()
    cursor.close()

def teaches_sheet(data):
    #class_section, sem, year, course_id, first_name, last_name
    cursor = conn.cursor()

    class_section = []
    sem = []
    year = []
    course_id = []
    first_name = []
    last_name = []

    #checking month/year
    if (data["Start/End Date"][0][1] == "9"):
        sems = "Fall"
    else:
        sems = "Spring"
    years = data["Start/End Date"][0][-4:]

    for index, row in data.iterrows():
      if(pd.isnull(row["Primary Instructor"])):
          class_section.append(row["Class Section"])
          course_id.append(row["Course ID"])
          sem.append(sems)
          year.append(years)
          first_name.append("N/A")
          last_name.append("N/A")
      elif("&" in row["Primary Instructor"]):
          names = row["Primary Instructor"].split(" & ")
          for i in names:
              class_section.append(row["Class Section"])
              course_id.append(row["Course ID"])
              sem.append(sems)
              year.append(years)
              name = i.split(",")
              if(len(name) > 1):
                  first_name.append(name[1])
              else:
                  first_name.append("N/A")
              if(name[0]):
                  last_name.append(name[0])
              else:
                  last_name.append("N/A")
      else:
          class_section.append(row["Class Section"])
          course_id.append(row["Course ID"])
          sem.append(sems)
          year.append(years)
          #add instructor
          name = row["Primary Instructor"].split(",")
          first_name.append(name[1])
          last_name.append(name[0])

    df = pd.DataFrame({"class_section":class_section, "sem":sem, "year":year, "course_id":course_id, "first_name":first_name, "last_name":last_name})
    for index, row in df.iterrows():
        query = 'INSERT INTO teaches(class_section, sem, year, course_id, first_name, last_name) VALUES (%s, %s, %s, %s, %s, %s) ON DUPLICATE KEY UPDATE class_section = VALUES(class_section), sem = VALUES(sem), year = VALUES(year), course_id = VALUES(course_id), first_name = VALUES(first_name), last_name = VALUES(last_name)'
        cursor.execute(query, (row["class_section"], row["sem"], row["year"], row["course_id"], row["first_name"], row["last_name"]))

    conn.commit()
    cursor.close()

def cross_listed_sheet(data):
    #class_section_u, sem_u, year_u, course_id_u, class_section_g, sem_g, year_g, course_id_g
    #match with title, professor
    cursor = conn.cursor()

    class_section_u =[]
    sem_u = []
    year_u = []
    course_id_u = []
    class_section_g = []
    sem_g = []
    year_g = []
    course_id_g = []

    undergrad = []

    #checking month/year
    if (data["Start/End Date"][0][1] == "9"):
        sems = "Fall"
    else:
        sems = "Spring"
    years = data["Start/End Date"][0][-4:]

    #grab data that is crosslisted
    for index, row in data.iterrows():
        if(row["Combined Status"] == "Combin" and "UY" in row["Subject/Catalog#"]):
            undergrad.append(row)

    #go through each data and find the crosslisted part
    for under in undergrad:
        for index, row in data.iterrows():
            if(under["Class Title"].lower() == "computer security" and row["Class Title"].lower() == "information security & privacy" and under["Primary Instructor"] == row["Primary Instructor"] and "GY" in row["Subject/Catalog#"] and under["Days"] == row["Days"] and under["Time"] == row["Time"]):
                class_section_u.append(under["Class Section"])
                sem_u.append(sems)
                year_u.append(years)
                course_id_u.append(under["Course ID"])

                class_section_g.append(row["Class Section"])
                sem_g.append(sems)
                year_g.append(years)
                course_id_g.append(row["Course ID"])

            elif(under["Class Title"].lower() == "interactive computer graphics" and row["Class Title"].lower() == "interact comp graphics" and under["Primary Instructor"] == row["Primary Instructor"] and "GY" in row["Subject/Catalog#"] and under["Days"] == row["Days"] and under["Time"] == row["Time"]):
                class_section_u.append(under["Class Section"])
                sem_u.append(sems)
                year_u.append(years)
                course_id_u.append(under["Course ID"])

                class_section_g.append(row["Class Section"])
                sem_g.append(sems)
                year_g.append(years)
                course_id_g.append(row["Course ID"])
            elif(under["Class Title"].lower() == row["Class Title"].lower() and under["Primary Instructor"] == row["Primary Instructor"] and "GY" in row["Subject/Catalog#"] and under["Days"] == row["Days"] and under["Time"] == row["Time"]):

                class_section_u.append(under["Class Section"])
                sem_u.append(sems)
                year_u.append(years)
                course_id_u.append(under["Course ID"])

                class_section_g.append(row["Class Section"])
                sem_g.append(sems)
                year_g.append(years)
                course_id_g.append(row["Course ID"])

    df = pd.DataFrame({"class_section_u":class_section_u, "sem_u":sem_u, "year_u":year_u, "course_id_u":course_id_u, "class_section_g": class_section_g, "sem_g":sem_g, "year_g":year_g, "course_id_g":course_id_g})
    for index, row in df.iterrows():
        query = 'INSERT INTO crosslisted(class_section_u, sem_u, year_u, course_id_u, class_section_g, sem_g, year_g, course_id_g) VALUES (%s, %s, %s, %s, %s, %s, %s, %s) ON DUPLICATE KEY UPDATE class_section_u = VALUES(class_section_u), sem_u = VALUES(sem_u), year_u = VALUES(year_u), course_id_u = VALUES(course_id_u), class_section_g = VALUES(class_section_g), sem_g = VALUES(sem_g), year_g = VALUES(year_g), course_id_g = VALUES(course_id_g)'
        cursor.execute(query, (row["class_section_u"], row["sem_u"], row["year_u"], row["course_id_u"], row["class_section_g"], row["sem_g"], row["year_g"], row["course_id_g"]))

    conn.commit()
    cursor.close()

def section_sheet(data):
    #class_section, sem, year course_id, room_num, building, day, timest, classnbr, class_type, enroll_cap, enroll_tot, waitlist_cap, class_topic, waitlist_tot, consent
    cursor = conn.cursor()

    sem = []
    year = []

    room_num, building = split_classroom(data)

    #checking month/year
    for index, row in data.iterrows():
        if (data["Start/End Date"][0][1] == "9"):
            sem.append("Fall")
        else:
            sem.append("Spring")
        year.append(data["Start/End Date"][0][-4:])

        data["Enrollment Capacity"] = data["Enrollment Capacity"].astype(str)
        data["Enrollment Total"] = data["Enrollment Total"].astype(str)
        data["Class Wait List Cap"] = data["Class Wait List Cap"].astype(str)
        data["Class Topic"] = data["Class Topic"].astype(str)
        data["Wait List Total"] = data["Wait List Total"].astype(str)
        data["Consent"] = data["Consent"].astype(str)

        if(row["Enrollment Capacity"] == "nan"):
            data.set_value(index, "Enrollment Capacity", "N/A")
        if(row["Enrollment Total"] == "nan"):
            data.set_value(index, "Enrollment Total", "N/A")
        if(pd.isnull(row["Class Wait List Cap"])):
            data.set_value(index, "Class Wait List Cap", "N/A")
        if(pd.isnull(row["Class Topic"])):
            data.set_value(index, "Class Topic", "N/A")
        if(pd.isnull(row["Wait List Total"])):
            data.set_value(index, "Wait List Total", "N/A")
        if(row["Consent"] == "nan"):
            data.set_value(index, "Consent", "No")
        if(pd.isnull(row["Days"])):
            data.set_value(index, "Days", "N/A")

    df = pd.DataFrame({"class_section":data["Class Section"], "sem":sem, "year":year, "course_id":data["Course ID"], "room_num":room_num, "building":building, "day":data["Days"], "timest":data["Time"], "classnbr":data["Class Nbr"],
                       "class_type":data["Component"], "enroll_cap":data["Enrollment Capacity"], "enroll_tot":data["Enrollment Total"], "waitlist_cap":data["Class Wait List Cap"],
                       "class_topic":data["Class Topic"], "waitlist_tot":data["Wait List Total"], "consent":data["Consent"]})
    for index, row in df.iterrows():
        query = 'INSERT INTO section(class_section, sem, year, course_id, room_num, building, days, timest, classnbr, class_type, enroll_cap, enroll_tot, waitlist_cap, waitlist_tot, class_topic, consent) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) ON DUPLICATE KEY UPDATE class_section = VALUES(class_section), sem = VALUES(sem), year = VALUES(year), course_id = VALUES(course_id), room_num = VALUES(room_num), building = VALUES(building), days = VALUES(days), timest = VALUES(timest), classnbr = VALUES(classnbr), class_type = VALUES(class_type), enroll_cap = VALUES(enroll_cap), enroll_tot = VALUES(enroll_tot), waitlist_cap = VALUES(waitlist_cap), waitlist_tot = VALUES(waitlist_tot), class_topic = VALUES(class_topic), consent = VALUES(consent)'
        cursor.execute(query, (row["class_section"], row["sem"], row["year"], row["course_id"], row["room_num"], row["building"], row["day"], row["timest"], row["classnbr"], row["class_type"], row["enroll_cap"], row["enroll_tot"], row["waitlist_cap"], row["waitlist_tot"], row["class_topic"], row["consent"]))

    conn.commit()
    cursor.close()

def run_sheets(data):
    course_sheet(data)
    classroom_sheet(data)
    instructor_sheet(data)
    time_sheet(data)
    section_sheet(data)
    teaches_sheet(data)
    cross_listed_sheet(data)

def main():
    print("Select 1 to run all files")
    print("Select 2 to run latest file")
    choice = input("What would you like to do? ")

    dirs = os.listdir(os.getcwd()) #get current directory and list
    if(choice == "1"):
        for file in dirs:
            if(file != "automate.py"):
                print(file)
                data = pd.read_excel(file)
                run_sheets(data)
                print("done")
    elif(choice == "2"):
        latest_file = max(dirs, key=os.path.getctime)
        print(latest_file)
        data = pd.read_excel(latest_file)
        run_sheets(data)
        print("done")

main()


import json
from abc import abstractmethod
from typing import Tuple

from sommelier.utils import StringUtils

def expand_nested_keys(payload):
    # To represent json in flattened form use `.` for specifying nested fields
    # * Given json:
    #
    # {name: Bob, education: {level: bachelor, grade: good}}
    #
    # * Flattened form:
    #
    # name: Bob
    # education.level: bachelor
    # eduction.grade: good

    nested_keys = []
    for k in payload:
        if '.' in k:
            nested_keys.append(k)

    for k in nested_keys:
        value = payload[k]
        current_obj = payload
        # Zoom into the key which will create new objects or reuse existing
        key_list = k.split('.')
        for i in range(len(key_list)):
            subkey = key_list[i]
            if subkey in current_obj:
                # reuse existing object and keep zooming
                current_obj = current_obj[subkey]
            else:
                if i == len(key_list) - 1:
                    # we reached the end and need to save value
                    current_obj[subkey] = value
                else:
                    # we are still zooming
                    current_obj[subkey] = {}
                current_obj = current_obj[subkey]

        # nobody needs fully qualified key at the end of this procedure
        del payload[k]

    return _convert_indexed_obj_to_arr(payload)

def _convert_indexed_obj_to_arr(payload):
    if not isinstance(payload, dict):
        return payload
    arr = []
    arr_key = None
    for k in payload:
        if StringUtils.is_array(k):
            arr_key = k
            arr.append(payload[k])
        else:
            payload[k] = _convert_indexed_obj_to_arr(payload[k])

    if arr_key is not None:
        return arr
    else:
        return payload

if __name__ == '__main__':
    def display(data):
        print(json.dumps(data, ensure_ascii=False, sort_keys=True, indent=4))

    def perform_check(given, expected):
        result = expand_nested_keys(given)
        ok = (expected == result)
        if not ok:
            display(expected)
            display(result)
        else:
            print("Successful")
    test_cases = [
        {
            "given": {
                "data.[0].user": {
                    "name": "John"
                },
                "data.[1].user": {
                    "name": "Bob"
                }
            },
            "expected": {
                "data": [
                    {"user": {"name": "John"}},
                    {"user": {"name": "Bob"}}
                ]
            }
        },
        {
            "given": {
                "[0]": 1,
                "[1]": 2,
            },
            "expected": [1, 2]
        },
        {
            "given": {
                "data.first.[0]": "apple",
                "data.first.[1]": "banana",
                "data.second.[0]": "kiwi",
                "data.second.[1].box": "carrots",
            },
            "expected": {
                "data": {
                    "first": [
                        "apple",
                        "banana",
                    ],
                    "second": [
                        "kiwi",
                        {"box": "carrots"},
                    ]
                }
            }
        }
    ]

    for v in test_cases:
        perform_check(v["given"], v["expected"])

class TableXDimensions(object):
    # Represents parsed Behave Table

    def __init__(self, data) -> None:
        self.data = data

    @abstractmethod
    def rows(self):
        # Can iterate over each row
        pass

class Table2D(TableXDimensions):
    # Contains 2 columns

    def __init__(self, data) -> None:
        super().__init__(data)

    def rows(self) -> Tuple:
        payload = dict(self.data)
        for k, val in payload.items():
            if isinstance(val, str):
                payload[k] = StringUtils.try_convert_num(val)
            elif isinstance(val, list):
                payload[k] = [StringUtils.try_convert_num(x) for x in val]
            else:
                payload[k] = val
        for row in payload.items():
            yield str(row[0]), row[1]

    def dict(self) -> dict:
        # convert from CSV notation to JSON aka Python dictionary
        payload = {}
        for key, val in self.rows():
            payload[key] = val

        return expand_nested_keys(payload)

class Table1D(TableXDimensions):
    # Contains 1 column

    def __init__(self, data) -> None:
        super().__init__(data)

    def rows(self) -> str:
        for row in self.data:
            yield str(row[0])

    def list(self) -> list:
        return self.data


# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right
class Solution:
    def dfs(self,root):
        if not root: return 0
        leftsum=max(self.dfs(root.left),0)
        rightsum=max(self.dfs(root.right),0)
        self.maxsum=max(self.maxsum,leftsum+rightsum+root.val)
        return root.val+max(leftsum,rightsum)

    def maxPathSum(self, root: Optional[TreeNode]) -> int:
        self.maxsum=float('-inf')
        self.dfs(root)
        return self.maxsum


from typing import List
from collections import Counter

def get_top_k_recommended_friends(user_id: int, cutoff_k: int) -> List[int]:
    # Step 1: Get the list of friends for the given user_id
    user_friends = set(get_friends(user_id))

    # Step 2: Get the list of friends for each of the user_id's friends
    mutual_friends = []
    for friend_id in user_friends:
        friend_friends = set(get_friends(friend_id))
        # Step 3: Calculate the number of mutual friends between each of the user_id's friend's friends and the original user_id
        mutual = friend_friends.intersection(user_friends)
        for mutual_friend in mutual:
            if mutual_friend != user_id and mutual_friend not in user_friends:
                mutual_friends.append(mutual_friend)

    # Step 4: Sort the list of recommended friends based on the number of mutual friends in descending order
    sorted_mutual_friends = [friend_id for friend_id, count in Counter(mutual_friends).most_common()]

    # Step 5: Return the top k recommended friends
    return sorted_mutual_friends[:cutoff_k]

def get_friends(user_id: int) -> List[int]:
    # Assuming that the social network is stored in a dictionary of user_ids and their friends
    social_network = {
        "Ahmet": ["Ekrem", "Sabri", "Messi"],
        "Ekrem": ["Ahmet", "Sabri"],
        "Sabri": ["Ahmet", "Ekrem", "Messi"],
        "Messi": ["Ahmet", "Sabri"],
        "Hasan": ["Messi", "Terim"],
        "Terim": ["Hasan"]
    }

    return social_network[user_id]

user_id = "Sabri"
cutoff_k = 2
user_friends = get_friends(user_id)
recommended_friends = get_top_k_recommended_friends(user_id, cutoff_k)
print(user_friends)
print(recommended_friends)

"""
Step 1 -) Get friends list for given user_id.

  O(Friends), where Friends is the number of friends of user_id.

Step 2-) Get the list of friends for each of the user_id's friends.

    O(Friends x N ), where N is the average number of friends for each user in the social network.

Step 3-) Calculate the number of mutual friends between each of the user_id's friend's friends and the original user_id.

	O(Friends x N).

Step 4-) Sort the list of recommended friends based on the number of mutual friends in descending order.

	O(N x log N), where N is the length of the mutual_friends list.

Step 5-) Return the top k recommended friends.

	O(K), where K is the recommended friends parameter.

(this x is multiplication). Complexity of the function is O(Friends x N + N x log N + K), space complexity of the function is O(Friends x N + N).

"""


import os
import re
from datetime import datetime
from copy import deepcopy

from openpyxl import Workbook
from scrapy.selector import Selector

def clean(text):
    if isinstance(text, (int, float)):
        return text
    if not text:
        return ''
    for c in ['\r\n', '\n\r', u'\n', u'\r', u'\t', u'\xa0', '\\']:
        text = text.replace(c, ' ')
    return re.sub(' +', ' ', text).strip()

class ParseProductsXmlScript:
    # txt / xml files directory
    files_directory = 'xml_files'

    # Output Results CSV file in which transaction records stored
    transactions_fp = 'toptel_products.csv'

    csv_headers = [
        "product_id", "name", "ean", "categories", "image_urls", "description", "link",
    ]

    def __init__(self):
        super().__init__()
        self.csv_writer = self.get_csv_writer()
        self.wb = Workbook()
        self.sheet = self.wb.active
        self.sheet_row_index = 1
        self.create_xls_write_headers()

        for fp in self.get_files():
            try:
                print(f"Parsing file: {fp}")
                self.parse_xml(fp)
            except Exception as exc:
                print(f"Exception while parsing file-> {fp}\n{exc}")

        self.wb.save(f"toptel_products.xlsx")

    def parse_xml(self, xml_path):
        xml_data = open(xml_path, encoding='utf-8').read()
        raw_xml = xml_data.replace('<![CDATA[', '').replace(']]>', '')
        selector = Selector(text=raw_xml)

        for trans_s in selector.css('product'):
            try:
                if not trans_s or not self.get_sel_text(trans_s, 'product_id'):
                    continue
                product = self.get_item(trans_s)
                self.write_xls(product)
                # self.write_to_csv(product)
            except Exception as err:
                print(err)
            a = 0

    def get_item(self, trans_s):
        item = {}
        item["product_id"] = self.get_sel_text(trans_s, 'product_id')
        item["name"] = self.get_sel_text(trans_s, 'name')
        item['ean'] = self.get_sel_text(trans_s, 'ean')
        item["categories"] = self.get_categories(trans_s)
        item["description"] = self.get_description(trans_s)
        item["image_urls"] = self.get_image_urls(trans_s)
        item["link"] = re.findall(r'link>(.*)', trans_s.get())[0].split('<')[0]

        if '85' == item["product_id"]:
            a = 0
        return item

    def get_sel_text(self, trans_s, css, default=None):
        if default is None:
            default = ''
        return (trans_s.css(f'{css}::text').get('') or trans_s.css(f'{css.lower()}::text').get('')).strip() or default

    def get_attr_val(self, trans_s, tag, attr, default=None):
        if default is None:
            default = ""
        return trans_s.css(f'{tag}::attr({attr})').get() or \
               trans_s.css(f'{tag.lower()}::attr({attr.lower()})').get() or default

    def get_attr_all_vals(self, trans_s, tag, attr, default=None):
        if default is None:
            default = []
        vals = trans_s.css(f'{tag}::attr({attr})').getall() or \
               trans_s.css(f'{tag.lower()}::attr({attr.lower()})').getall() or default
        return ", ".join(f'"{v}"' for v in vals if clean(v))

    def write_to_csv(self, item):
        row = '$'.join('"{}"'.format(item.get(h, '')) for h in self.csv_headers) + '\n'
        self.csv_writer.write(row)
        print("Record inserted into CSV file.")
        print(item)

    def get_csv_writer(self):
        file = open(self.transactions_fp, mode='w', encoding='utf-8')
        file.write('$'.join(h for h in self.csv_headers) + '\n')
        return file

    def get_files(self):
        files = []

        for file_path in os.listdir(self.files_directory):
            if '.xml' not in file_path:
                continue
            file_path = self.files_directory + '/' + file_path
            files.append(file_path)

        return files

    def get_categories(self, trans_s):
        return ", ".join([clean(e) for e in trans_s.css('cat_name::text').getall()])

    def get_image_urls(self, trans_s):
        urls = [e for e in trans_s.css('photo::text, photo ::text').getall() if 'http:' in e] \
               or re.findall(r'photo>(.*)<', trans_s.get())
        if not urls:
            urls.append(f"https://toptel.com/files/fotob/product-{self.get_sel_text(trans_s, 'product_id')}.jpg")
        return ", ".join(urls)

    def get_description(self, trans_s):
        return clean(trans_s.css('desc').get())

    def create_xls_write_headers(self):
        for index, value in enumerate(self.csv_headers):
            self.sheet.cell(row=self.sheet_row_index, column=index + 1).value = value
        self.sheet_row_index += 1

    def write_xls(self, item):
        for idx, key in enumerate(self.csv_headers):
            self.sheet.cell(row=self.sheet_row_index, column=idx + 1).value = item.get(key, '')

        self.sheet_row_index += 1
        print(f"Product written in excel \n {item}")

# main function that is entry point of python script
if __name__ == "__main__":
    ParseProductsXmlScript()


#!/usr/bin/env python
"""
Test module for TwoPhaseFlow
"""
import pytest
import tables
import numpy as np
import proteus.defaults
from proteus import Context
from proteus import default_so
from proteus.iproteus import *
import os
import sys
Profiling.logLevel=1
Profiling.verbose=True

class TestTwoPhaseFlow(object):

    def setup_method(self,method):
        self._scriptdir = os.path.dirname(__file__)
        self.path = proteus.__path__[0]+"/tests/TwoPhaseFlow/"

    def teardown_method(self, method):
        """ Tear down function """
        FileList = ['marin.h5','marin.xmf'
                    'moses.h5','moses.xmf'
                    'damBreak.h5','damBreak.xmf'
                    'TwoDimBucklingFlow.h5','TwoDimBucklingFlow.xmf'
                    'filling.h5','filling.xmf'
                    ]
        for file in FileList:
            if os.path.isfile(file):
                os.remove(file)
            else:
                pass

    def compare_vs_saved_files(self,name,write=False):
        actual = tables.open_file(name+'.h5','r')

        expected_path = 'comparison_files/' + 'comparison_' + name + '_phi_t2.csv'
        #write comparison file
        if(write):
            np.array(actual.root.phi_t2).tofile(os.path.join(self._scriptdir, expected_path),sep=",")
        np.testing.assert_almost_equal(np.fromfile(os.path.join(self._scriptdir, expected_path),sep=","),np.array(actual.root.phi_t2).flatten(),decimal=6)

        expected_path = 'comparison_files/' + 'comparison_' + name + '_velocity_t2.csv'
        #write comparison file
        if(write):
            np.array(actual.root.velocity_t2).tofile(os.path.join(self._scriptdir, expected_path),sep=",")
        np.testing.assert_almost_equal(np.fromfile(os.path.join(self._scriptdir, expected_path),sep=","),np.array(actual.root.velocity_t2).flatten(),decimal=6)

        actual.close()

    # *** 2D tests *** #
    def test_risingBubble(self): #uses structured triangle mesh
        os.system("parun --TwoPhaseFlow --path " + self.path + " "
                  "risingBubble.py -l5 -v -C 'final_time=0.1 dt_output=0.1 refinement=1'")
        self.compare_vs_saved_files("risingBubble")

    def test_damBreak(self):
        os.system("parun --TwoPhaseFlow --path " + self.path + " "
                  "damBreak.py -l5 -v -C 'final_time=0.1 dt_output=0.1 he=0.1'")
        self.compare_vs_saved_files("damBreak")

    def test_damBreak_hotstart(self):
        os.system("parun --TwoPhaseFlow --path " + self.path + " "
                  "damBreak.py -l5 -v -H -C 'final_time=0.1 dt_output=0.1 he=0.1 hotstart=True'")

    @pytest.mark.skipif(sys.platform == "darwin", reason="does not run on macOS")
    def test_TwoDimBucklingFlow(self):
        os.system("parun --TwoPhaseFlow --path " + self.path + " "
                  "TwoDimBucklingFlow.py -l5 -v -C 'final_time=0.1 dt_output=0.1 he=0.09'")
        self.compare_vs_saved_files("TwoDimBucklingFlow")

    def test_fillingTank(self):
        os.system("parun --TwoPhaseFlow --path " + self.path + " "
                  "fillingTank.py -l5 -v -C 'final_time=0.02 dt_output=0.02 he=0.01'")
        self.compare_vs_saved_files("fillingTank")

    # *** 3D tests *** #
    @pytest.mark.skipif(sys.platform == "darwin", reason="does not run on macOS")
    def test_marin(self):
        os.system("parun --TwoPhaseFlow --path " + self.path + " "
                  "marin.py -l5 -v -C 'final_time=0.1 dt_output=0.1 he=0.5'")
        self.compare_vs_saved_files("marin")

    def test_moses(self):
        os.system("parun --TwoPhaseFlow --path " + self.path + " "
                  "moses.py -l5 -v -C 'final_time=0.1 dt_output=0.1 he=0.5'")
        self.compare_vs_saved_files("moses")

    def test_damBreak_genPUMI(self):
        os.system("parun --TwoPhaseFlow --genPUMI --path " + self.path + " "
                  "damBreak.py -l5 -v -C 'final_time=0.1 dt_output=0.1 he=0.1'")

    def test_damBreak_runPUMI(self):
        os.system("parun --TwoPhaseFlow --path " + self.path + " "
                  "damBreak_PUMI.py -l5 -v -C 'final_time=0.1 dt_output=0.1 he=0.1 adapt=0'")
        self.compare_vs_saved_files("damBreak_PUMI")

    def test_damBreak_pseudo_CLSVOF(self):
        os.system("parun --TwoPhaseFlow --path " + self.path + " "
                  "damBreak_PUMI.py -l5 -v -C 'final_time=0.1 dt_output=0.1 he=0.1 adapt=1'")
        self.compare_vs_saved_files("damBreak_PUMI")


import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(
    name="pyservice",  # Replace with your own username
    version="0.0.11",
    author="Attila Domokos",
    author_email="adomokos@gmail.com",
    description="Series of Actions with an emphasis on simplicity.",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/adomokos/pyservice",
    packages=setuptools.find_packages(),
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ],
    python_requires=">=3.6",
)