class FileManager():
    def __init__(self):
        pass

    def WriteToFile(self, info):
        self.info = info
        with open(r"PATH\todolist.txt", "a") as file:
            file.write(self.info + "\n")

    def ReadFile(self):
        with open(r"PATH\todolist.txt", "r") as file:
            self.items = file.read().strip()
        return self.items

class UserTalk(FileManager):
    def __init__(self):
        super().__init__()
        print("Welcome to the To-Do List Application done with OOPS Python")
        print("Enter 1 - Add a Task \n Enter 2 - View Tasks")
        self.ask = input("")
        if self.ask.strip() == '1':
            self.AddTask()

        elif self.ask.strip() == '2':
            self.ViewTasks()

        else:
            print("Only Enter 1 or 2")

    def AddTask(self):
        print("Enter the Task in Brief")
        self.task = input("")
        self.task = self.task.strip()
        self.WriteToFile(self.task)
        print("Task Added!")

    def ViewTasks(self):
        print("Here are your Tasks!")
        print("---")
        print(self.ReadFile())
        print("---")

obj = UserTalk()


#The same exercice as 4.7

import turtle
import random

wn = turtle.Screen()

drunkpirate = turtle.Turtle()

randomexperience = [160, -43, 270, -97, -43, 200, -940, 17, -86]

def randomwalk(n):
    for i in range(0,7):
        drunkpirate.left(randomexperience[n])
        drunkpirate.forward(100)

for element in randomexperience:
    randomwalk(element)

wn.exitonclick()


from django.shortcuts import render
from .forms import GroceryForm
from django.http import JsonResponse
# Create your views here.
#def grocery_list(request):
#    return render(request, 'grocery_list_app/grocery_list.html', {})

from .models import FoodPrices

def submit_grocery_list(request):

    if request.method == "POST":
        form = GroceryForm(request.POST)

    else:
        form = GroceryForm()

    #this view will actually be coming from the map part, and will redirect to the grocery list page
    #make a dictionary with dollar sign info and list of foods available at that type of store?
    #add that dictionary to the render thing
    #somehow edit the dropdown menu on the form based on the list of foods...

    return render(request, 'grocery_list_app/grocery_list_2.html', {'form': form})

def cash_register(request):
    food_id = request.GET.get('food_id', None)
    data = {
            'food_price': FoodPrices.objects.get(id=food_id).food_price,
            'food_quantity': FoodPrices.objects.get(id=food_id).food_quantity,
            'food_name': FoodPrices.objects.get(id=food_id).food_name
            }

    #probably add dollar signs to this dictionary
    #e.g. 'dollar_sign': 1

    return JsonResponse(data)


#
# trainer.py
# codes for training GAN
#
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import time
import warnings

from tqdm import tqdm
from torch.utils.data import DataLoader
from torchvision.utils import make_grid

from util.components import *
from util.loss import loss_gen_bce, loss_dis_bce, Rotate3dLoss
from util.camera_param import CameraParam
from util.save_results import generate_sample_rgbd, convert_batch_images_rgbd, save_batch_sample_rgbd, save_loss_graph
from util.osgan_module import get_gradient_ratios, GradientScaler
from util import fid_score

torch.multiprocessing.set_sharing_strategy('file_system')

class TrainerPGGAN:
    """
    class of PGGAN for generating RGB images
    """
    def __init__(self, config_train, config_transform=None, device=torch.device('cuda')):
        self.gen = config_train['generator']
        self.dis = config_train['discriminator']

        self.optim_g = config_train['optim_gen']
        self.optim_d = config_train['optim_dis']

        self.dataset = config_train['dataset']

        self.schedule = config_train['schedule']
        self.latent_size = config_train['latent_size']
        self.lambda_gp = config_train['lambda_gp']
        self.in_res = config_train['in_res']
        self.out_res = config_train['out_res']
        self.iteration = config_train['iteration']

        self.checkpoint_dir = config_train['root_path'] + 'checkpoint/'
        self.out_dir = config_train['root_path'] + 'output/'
        self.weight_dir = config_train['root_path'] + 'weight/'
        self.loss_dir = config_train['root_path'] + 'loss/'

        self.fid = config_train['fid']

        self.osgan = config_train['osgan']

        self.rgbd = config_train['rgbd']

        if self.rgbd:
            self.start_rotation = config_transform['start_rotation']
            self.start_occlusion_aware = config_transform['start_occlusion_aware']
            self.rotate_3d_loss = Rotate3dLoss(lambda_geometric=config_transform['lambda_geometric'])
            self.lambda_depth = config_transform['lambda_depth']
            self.depth_min = config_transform['depth_min']
            self.camera_param = CameraParam(
                config_transform['train_x_rotate'],
                config_transform['train_y_rotate'],
                config_transform['train_z_rotate'],
                config_transform['train_x_translate'],
                config_transform['train_y_translate'],
                config_transform['train_z_translate'],
                device=device
            )

            self.test_x_rotate = config_transform['test_x_rotate']
            self.test_y_rotate = config_transform['test_y_rotate']
            self.test_z_rotate = config_transform['test_z_rotate']
            self.test_x_translate = config_transform['test_x_translate']
            self.test_y_translate = config_transform['test_y_translate']
            self.test_z_translate = config_transform['test_z_translate']

        self.device = device
        self.fixed_latent = self.make_hidden(16)

    def make_hidden(self, batch_size):
        z = torch.normal(0, 1, size=(batch_size, self.latent_size, 1, 1))
        z /= torch.sqrt(torch.sum(z * z, dim=1, keepdims=True) / self.latent_size + 1e-8)
        return z.to(self.device)

    def train(self):
        warnings.filterwarnings(action='ignore')
        # initial definition
        epoch_time_each = np.zeros(self.iteration)
        epoch_time_accum = np.zeros(self.iteration)
        running_time_accum = 0.

        running_loss_d = 0.
        epoch_losses_d = np.zeros(self.iteration)
        running_loss_g = 0.
        epoch_losses_g = np.zeros(self.iteration)
        if self.osgan:
            running_loss_p = 0.
            epoch_losses_p = np.zeros(self.iteration)
        iter_num = 0

        if self.fid:
            fids = np.zeros(self.iteration)

        if self.in_res == 4:
            c = 0
            self.gen.depth = 1
            self.dis.depth = 1
        else:
            c = int(np.log2(self.in_res) - 3)
            self.gen.depth = c + 2
            self.dis.depth = c + 2

        batch_size = self.schedule[1][c]
        growing = self.schedule[2][c]

        data_loader = DataLoader(dataset=self.dataset, batch_size=batch_size, shuffle=True, num_workers=8)

        tot_iter_num = len(self.dataset) / batch_size

        size = 2 ** (self.gen.depth + 1)
        print("Output Resolution: %d x %d" % (size, size))

        for epoch in range(1, self.iteration + 1):
            use_rotate = False
            if self.osgan:
                scaler = GradientScaler.apply

            if self.rgbd:
                if epoch >= self.start_rotation:
                    use_rotate = True

            self.gen.train()

            epoch_loss_d = 0.
            epoch_loss_g = 0.
            if self.osgan:
                epoch_loss_p = 0.

            if epoch - 1 in self.schedule[0]:
                if 2 ** (self.gen.depth + 1) < self.out_res:
                    c = self.schedule[0].index(epoch - 1)
                    batch_size = self.schedule[1][c]
                    growing = self.schedule[2][c]
                    data_loader = DataLoader(dataset=self.dataset, batch_size=batch_size, shuffle=True, num_workers=8)
                    tot_iter_num = len(self.dataset) / batch_size
                    self.gen.growing_net(growing * tot_iter_num)
                    self.dis.growing_net(growing * tot_iter_num)
                    size = 2 ** (self.gen.depth + 1)
                    print("Output Resolution: %d x %d" % (size, size))

            print("epoch: %i/%i, batch size: %i" % (int(epoch), int(self.iteration), int(batch_size)))
            databar = tqdm(data_loader)

            start = time.time()
            for i, samples in enumerate(databar):

                # prepare samples
                if size != self.out_res:
                    samples = F.interpolate(samples[0], size=size).to(self.device)
                else:
                    samples = samples[0].to(self.device)

                # prepare random_camera_matrix
                thetas = None
                if self.rgbd:
                    thetas = self.camera_param.get_sample_param(samples.size(0))
                    random_camera_matrix = self.camera_param.get_ex_matrices(thetas)
                    thetas = torch.reshape(
                        # only use x,y rotations with cos, sin
                        # torch.cat([torch.cos(thetas[:, :3]), torch.sin(thetas[:, :3]), thetas[:, 3:]], dim=1),
                        torch.cat([torch.cos(thetas[:, :2]), torch.sin(thetas[:, :2])], dim=1),
                        # (samples.size(0), 9, 1, 1)
                        (samples.size(0), 4, 1, 1)
                    )
                if not self.osgan:
                    # update D
                    self.optim_d.zero_grad()
                    if self.rgbd:
                        latent_z = torch.cat(
                            [self.make_hidden(samples.size(0) // 2)] * 2,
                            dim=0
                        )
                    else:
                        latent_z = self.make_hidden(samples.size(0))

                    x_fake = self.gen(latent_z, theta=thetas)
                    y_fake = self.dis(x_fake[:, :3].detach())
                    y_real = self.dis(samples)

                    # gradient penalty
                    eps = torch.rand(samples.size(0), 1, 1, 1, device=self.device)
                    eps = eps.expand_as(samples)
                    x_hat = eps * samples + (1 - eps) * x_fake[:, :3].detach()
                    x_hat.requires_grad = True
                    px_hat = self.dis(x_hat)
                    grad = torch.autograd.grad(outputs=px_hat.sum(), inputs=x_hat, create_graph=True)[0]
                    grad_norm = grad.view(samples.size(0), -1).norm(2, dim=1)
                    gradient_penalty = self.lambda_gp * ((grad_norm - 1) ** 2).mean()

                    # backpropagate D loss
                    loss_d, _, _ = loss_dis_bce(y_fake, y_real)
                    loss_d += gradient_penalty
                    assert not torch.isnan(loss_d.data)
                    loss_d.backward()
                    self.optim_d.step()

                    # update G
                    self.optim_g.zero_grad()
                    y_fake = self.dis(x_fake[:, :3])

                    loss_rotate = 0.
                    if use_rotate:
                        # 3d loss
                        loss_rotate, warped_dp = self.rotate_3d_loss(
                            x_fake[:samples.size(0) // 2],
                            random_camera_matrix[:samples.size(0) // 2],
                            x_fake[samples.size(0) // 2:],
                            random_camera_matrix[samples.size(0) // 2:],
                            epoch >= self.start_occlusion_aware
                        )

                        if self.lambda_depth > 0:
                            # depth regularization
                            loss_rotate += torch.mean(F.relu(self.depth_min - x_fake[:, -1]) ** 2) * self.lambda_depth

                        assert not torch.isnan(loss_rotate.data)
                        lambda_rotate = 2 if size <= self.out_res else 4
                        loss_rotate = loss_rotate * lambda_rotate

                    # backpropagate G loss
                    loss_g = torch.mean(loss_gen_bce(y_fake)) + loss_rotate
                    assert not torch.isnan(loss_g.data)
                    loss_g.backward()
                    self.optim_g.step()
                else:
                    if self.rgbd:
                        latent_z = torch.cat(
                            [self.make_hidden(samples.size(0) // 2)] * 2,
                            dim=0
                        )
                    else:
                        latent_z = self.make_hidden(samples.size(0))

                    x_fake = self.gen(latent_z, theta=thetas)
                    x_fake_neg = scaler(x_fake)
                    y_fake = self.dis(x_fake_neg[:, :3])
                    y_real = self.dis(samples)

                    # gradient penalty
                    eps = torch.rand(samples.size(0), 1, 1, 1, device=self.device)
                    eps = eps.expand_as(samples)
                    x_hat = eps * samples + (1 - eps) * x_fake[:, :3].detach()
                    x_hat.requires_grad = True
                    px_hat = self.dis(x_hat)
                    grad = torch.autograd.grad(outputs=px_hat.sum(), inputs=x_hat, create_graph=True)[0]
                    grad_norm = grad.view(samples.size(0), -1).norm(2, dim=1)
                    gradient_penalty = self.lambda_gp * ((grad_norm - 1) ** 2).mean()

                    # D loss
                    loss_d, real_loss, fake_loss = loss_dis_bce(y_fake, y_real)
                    loss_d += gradient_penalty
                    assert not torch.isnan(loss_d.data)

                    loss_rotate = 0.
                    if use_rotate:
                        # 3d loss
                        loss_rotate, warped_dp = self.rotate_3d_loss(
                            x_fake[:samples.size(0) // 2],
                            random_camera_matrix[:samples.size(0) // 2],
                            x_fake[samples.size(0) // 2:],
                            random_camera_matrix[samples.size(0) // 2:],
                            epoch >= self.start_occlusion_aware
                        )

                        if self.lambda_depth > 0:
                            # depth regularization
                            loss_rotate += torch.mean(F.relu(self.depth_min - x_fake[:, -1]) ** 2) * self.lambda_depth

                        assert not torch.isnan(loss_rotate.data)
                        lambda_rotate = 2 if size <= self.out_res else 4
                        loss_rotate = loss_rotate * lambda_rotate

                    # P loss
                    loss_g = loss_gen_bce(y_fake)
                    gamma = get_gradient_ratios(loss_g, fake_loss, y_fake)

                    grad_d_factor = 1. / (1. - gamma)
                    loss_pack_fake = fake_loss - loss_g
                    scaled_loss_pack_fake = loss_pack_fake * grad_d_factor
                    loss_pack = real_loss + torch.mean(scaled_loss_pack_fake) + gradient_penalty + loss_rotate
                    assert not torch.isnan(loss_pack.data)

                    GradientScaler.factor = gamma

                    # G loss
                    loss_g = torch.mean(loss_g) + loss_rotate
                    assert not torch.isnan(loss_g.data)

                    # backpropagate P loss
                    self.optim_d.zero_grad()
                    self.optim_g.zero_grad()

                    loss_pack.backward()

                    self.optim_d.step()
                    self.optim_g.step()

                running_loss_d += loss_d.item()
                running_loss_g += loss_g.item()

                epoch_loss_d += loss_d.item()
                epoch_loss_g += loss_g.item()

                if self.osgan:
                    running_loss_p += loss_pack.item()
                    epoch_loss_p += loss_pack.item()

                iter_num += 1

                # print current loss
                if i % 500 == 0:
                    running_loss_d /= iter_num
                    running_loss_g /= iter_num
                    if self.osgan:
                        running_loss_p /= iter_num
                    print('iteration: %d, gp: %.2f' % (i, gradient_penalty))
                    if not self.osgan:
                        discription = 'loss_d: %.3f   loss_g: %.3f' % (running_loss_d, running_loss_g)
                    else:
                        discription = 'loss_d: %.3f   loss_g: %.3f   loss_p: %.3f' \
                                      % (running_loss_d, running_loss_g, running_loss_p)
                    databar.set_description(discription)
                    iter_num = 0
                    running_loss_d = 0.
                    running_loss_g = 0.
                    if self.osgan:
                        running_loss_p = 0.
            epoch_time = time.time() - start
            print("elapsed_time (sec): %.3f" % epoch_time)

            # get total losses of one epoch
            epoch_losses_d[epoch - 1] = (epoch_loss_d / tot_iter_num)
            epoch_losses_g[epoch - 1] = (epoch_loss_g / tot_iter_num)
            if self.osgan:
                epoch_losses_p[epoch - 1] = (epoch_loss_p / tot_iter_num)

            # save elapsed time
            epoch_time_each[epoch - 1] = epoch_time
            running_time_accum += epoch_time
            epoch_time_accum[epoch - 1] = running_time_accum

            # get checkpoint and save + generate samples
            checkpoint = {'gen': self.gen.state_dict(),
                           'optim_g': self.optim_g.state_dict(),
                           'dis': self.dis.state_dict(),
                           'optim_d': self.optim_d.state_dict(),
                           'epoch_losses_d': epoch_losses_d,
                           'epoch_losses_g': epoch_losses_g,
                           'fixed_latent': self.fixed_latent,
                           'depth': self.gen.depth,
                           'alpha': self.gen.alpha
                           }
            gen_parameters = {
                'gen': self.gen.state_dict(),
                'depth': self.gen.depth,
                'alpha': self.gen.alpha,
                'out_res': self.out_res
            }

            with torch.no_grad():
                self.gen.eval()
                if self.fid:
                    real_batch = []
                    fake_batch = []

                    for i, (real, _) in enumerate(data_loader):
                        fid_latent = self.make_hidden(batch_size)
                        thetas = torch.reshape(
                            torch.tile(torch.tensor([1., 1., 0., 0.]), (batch_size, 1)), (batch_size, 4, 1, 1)
                        ).to(self.device)
                        fake = self.gen(fid_latent, theta=thetas)[:, :-1]
                        real_batch.append(real.type(torch.FloatTensor))
                        fake_batch.append(fake.type(torch.FloatTensor))

                        if i * batch_size > 2500:
                            break

                    real_batch = torch.cat(real_batch, dim=0)
                    fake_batch = torch.cat(fake_batch, dim=0)

                    score = fid_score.calculate_fid_given_batches(real_batch, fake_batch, batch_size=32)
                    fids[epoch - 1] = score
                    print("fid: %.5f" % score)

                    # save fid scores via iteration
                    plt.figure()
                    plt.plot(fids, '-', color='red', label='FID')
                    plt.xlabel('iteration')
                    plt.ylabel('FID')
                    plt.legend()
                    plt.tight_layout()
                    plt.savefig(self.loss_dir + 'fid_iteration')
                    plt.close()

                    # save fid scores via elapsed time
                    plt.figure()
                    plt.plot(epoch_time_accum, fids, '-', color='red', label='FID')
                    plt.xlabel('elapsed time')
                    plt.ylabel('FID')
                    plt.legend()
                    plt.tight_layout()
                    plt.savefig(self.loss_dir + 'fid_time')
                    plt.close()

                if epoch == self.iteration:
                    torch.save(checkpoint, self.checkpoint_dir + 'checkpoint_epoch_%d.pth' % epoch)
                    torch.save(gen_parameters, self.weight_dir + 'gen_weight_epoch_%d.pth' % epoch)
                if self.rgbd:
                    angle_range = 8
                    sample_x = generate_sample_rgbd(
                        self.gen, self.fixed_latent, self.test_y_rotate, angle_range=angle_range, device=self.device)
                    x, depth = convert_batch_images_rgbd(sample_x, angle_range)
                    save_batch_sample_rgbd(x, path=self.out_dir + 'size_%i_epoch_%d' % (size, epoch), depth=depth)
                else:
                    plt.figure()
                    out_imgs = self.gen(self.fixed_latent)
                    out_grid = make_grid(
                        out_imgs, normalize=True, nrow=4, scale_each=True, padding=int(0.5*(2**self.gen.depth))
                    ).permute(1, 2, 0)
                    plt.imshow(out_grid.cpu())
                    plt.savefig(self.out_dir + 'size_%i_epoch_%d' % (size, epoch))
                    plt.close()

                # save elapsed time via iteration
                plt.figure()
                plt.plot(epoch_time_each, '-', color='red', label='elapsed time')
                plt.xlabel('iteration')
                plt.ylabel('elapsed time')
                plt.legend()
                plt.tight_layout()
                plt.savefig(self.loss_dir + 'time_iteration')
                plt.close()

                # save loss graph
                if not self.osgan:
                    save_loss_graph(epoch_losses_d, epoch_losses_g, path=self.loss_dir + 'loss_iteration')
                    save_loss_graph(epoch_losses_d, epoch_losses_g, path=self.loss_dir + 'loss_time', x=epoch_time_accum)
                else:
                    save_loss_graph(
                        epoch_losses_d, epoch_losses_g, path=self.loss_dir + 'loss_iteration', p=epoch_losses_p
                    )
                    save_loss_graph(
                        epoch_losses_d, epoch_losses_g, path=self.loss_dir + 'loss_time', x=epoch_time_accum, p=epoch_losses_p
                    )

                # save all results in txt file
                txt = open(self.loss_dir + 'train_log.txt', 'at')
                if self.osgan:
                    if self.fid:
                        txt.write(
                            'epoch:%d,d_loss:%.6f,g_loss:%.6f,p_loss:%.6f,fid:%.6f,elapsed_time:%.6f\n' % (
                                epoch,
                                epoch_losses_d[epoch - 1],
                                epoch_losses_g[epoch - 1],
                                epoch_losses_p[epoch - 1],
                                fids[epoch - 1],
                                epoch_time_each[epoch - 1]
                            )
                        )
                    else:
                        txt.write(
                            'epoch:%d,d_loss:%.6f,g_loss:%.6f,p_loss:%.6f,elapsed_time:%.6f\n' % (
                                epoch,
                                epoch_losses_d[epoch - 1],
                                epoch_losses_g[epoch - 1],
                                epoch_losses_p[epoch - 1],
                                epoch_time_each[epoch - 1]
                            )
                        )
                else:
                    if self.fid:
                        txt.write(
                            'epoch:%d,d_loss:%.6f,g_loss:%.6f,fid:%.6f,elapsed_time:%.6f\n' % (
                                epoch,
                                epoch_losses_d[epoch - 1],
                                epoch_losses_g[epoch - 1],
                                fids[epoch - 1],
                                epoch_time_each[epoch - 1]
                            )
                        )
                    else:
                        txt.write(
                            'epoch:%d,d_loss:%.6f,g_loss:%.6f,elapsed_time:%.6f\n' % (
                                epoch,
                                epoch_losses_d[epoch - 1],
                                epoch_losses_g[epoch - 1],
                                epoch_time_each[epoch - 1]
                            )
                        )
                txt.close()


import turtle
import math
wdth = 800; hgth = 800; bgstring = "#ffffff"
red = "#cc0000"; green = "#00cc00"; blue = "#0000cc"

def grid(t):
	x = 0; y = 0
	while (x < 400):
		t.color(green)
		t.penup()
		t.goto(x,y)
		t.pendown()
		t.goto(x,y+400)
		x = x + 50
	x = 0; y = 0
	while (y < 400):
		t.penup()
		t.goto(x,y)
		t.pendown()
		t.goto(x+400,y)
		y = y + 50
	t.color('orange')
	style = ('Courior', 20, 'bold')
	t.write('Circle Circumference and Diameter', font=style, align='right')
	t.penup()
	t.goto(150,-50)
	t.pendown()
	t.write('Diameter', font=style, align='left')
	t.penup()
	t.goto(-130,180)
	t.pendown()
	t.write('Circumference', font=style, align='center')
	t.penup()

def plotCircles(t):
	d =  [12.8, 1.8, 19.8, 8.7]
	c =  [3*12.8,  3*1.8, 3*19.7, 3* 8.7]
	dsorted = sorted (d, key = float)
	csorted = sorted(c , key = float)
	t.goto(0,0)
	t.color(red)
	t.pendown()
	t.dot(3, blue)
	t.goto(dsorted[0],csorted[0])
	t.dot(3, blue)
	t.goto(dsorted[1],csorted[1])
	t.dot(3, blue)
	t.goto(dsorted[2],csorted[2])
	t.dot(3, blue)
	t.goto(dsorted[3],csorted[3])
	t.dot(3, blue)

def main():
	try:
		turtle.TurtleScreen._RUNNING = True
		# get wdth and hgth globally
		turtle.screensize(canvwidth=wdth, canvheight=hgth, bg=bgstring)
		print(turtle.Screen().screensize())
		w = turtle.Screen()
		t = turtle.Turtle()
		t.hideturtle()
		grid(t)
		plotCircles(t)
		w.exitonclick()
	finally:
		turtle.Terminator()

if __name__ == '__main__':
	main()


import torch
from torch import nn
from torch.utils.data import DataLoader
from utils import upload_paths
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score, roc_auc_score
from config_ import config
from models import Baseline
from datasets import ContrastiveFaceDataset
from models import ArcFaceLoss, ContrastiveSigmoidWrapper
from tqdm.auto import tqdm
from torch.amp import autocast

class ROC_AUC(nn.Module):
    def __init__(self) -> None:
        super().__init__()

    def forward(self, x: torch.Tensor, y: torch.Tensor):
        with torch.inference_mode():
            pred = torch.sigmoid(x).round().detach().cpu().numpy()
            y = y.detach().cpu().numpy()
        return roc_auc_score(y, pred)

class F1(nn.Module):
    def __init__(self) -> None:
        super().__init__()

    def forward(self, x, y):
        with torch.inference_mode():
            pred = torch.sigmoid(x).round().detach().cpu().numpy()
            y = y.detach().cpu().numpy()
        return f1_score(y, pred)

class Trainer:
    losses = {
        "arcface_loss": ArcFaceLoss,
        "modified_bce": ContrastiveSigmoidWrapper,
        "cosine_embedding_loss": nn.CosineEmbeddingLoss
    }

    def __init__(self) -> None:

        self.f1_score = F1()
        self.best_f1 = 0

        self.model = Baseline()
        self.model.to(config.device)
        self.optimizer = torch.optim.AdamW(self.model.parameters(),
                                            lr=config.lr, weight_decay=config.weight_decay)
        self.criterion = self.losses[config.loss]()

        dataset = upload_paths()
        train_data, val_data = train_test_split(dataset, test_size=config.validation_data_ratio)
        train_dataset = ContrastiveFaceDataset(img_paths=train_data, train=True)
        val_dataset = ContrastiveFaceDataset(img_paths=val_data, train=False)

        self.train_dataloader = DataLoader(train_dataset, batch_size=config.batch_size, shuffle=True)
        self.val_dataloader = DataLoader(val_dataset, batch_size=config.batch_size, shuffle=False)

    def measure_metrics(self, outputs, labels):
        return self.f1_score(outputs, labels)

    def fit(self):
        print(f"training starts on {config.device_str}")
        self.model.to(config.device)

        for epoch in range(1, config.num_epochs + 1):
            self.train_epoch(epoch)
            self.validate_epoch(epoch)

    def train_epoch(self, epoch):
        self.model.train()

        pbar = tqdm(
            enumerate(self.train_dataloader),
            total = len(self.train_dataloader),
            desc = f"Epoch(train) {epoch} ")
        running_loss = 0

        for idx, (img1, img2, label) in pbar:
            img1 = img1.to(config.device)
            img2 = img2.to(config.device)
            label = label.to(config.device)

            self.optimizer.zero_grad()

            with autocast(device_type=config.device_str):
                embedding1 = self.model(img1)
                embedding2 = self.model(img2)
                loss = self.criterion(embedding1, embedding2, label.float())

            running_loss += loss.item()

            pbar.set_postfix(
                dict(loss = round(running_loss / (idx + 1), 5)))

            loss.backward()
            nn.utils.clip_grad_norm_(self.model.parameters(), config.max_grad_norm)
            self.optimizer.step()

    def validate_epoch(self, epoch):
        self.model.eval()

        pbar = tqdm(
            enumerate(self.val_dataloader),
            total = len(self.val_dataloader),
            desc = f"Epoch(validation) {epoch} ")

        running_loss = 0
        running_f1 = 0

        for idx, (img1, img2, label) in pbar:
            img1 = img1.to(config.device)
            img2 = img2.to(config.device)
            label = label.to(config.device)

            with torch.no_grad():
                emb1 = self.model(img1)
                emb2 = self.model(img2)
                pred = (emb1 - emb2).pow(2).sum(1)

            running_loss += self.criterion(emb1, emb2, label.float()).item()

            f1 = self.measure_metrics(pred, label)
            running_f1 += f1

            pbar.set_postfix(
                dict(
                    f1 = round(running_f1 / (idx + 1), 5),
                    loss = round(running_loss/(idx + 1), 5)
                )
            )

            if running_f1 / (idx + 1) > self.best_f1:
                self.best_f1 = running_f1 / (idx + 1)
                torch.save(self.model.state_dict(), config.best_weights_path)
                print(f"saved model weights at: {config.best_weights_path}")


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

directory = "./test5_1"
frequency = 500
mi_estimation_method = "middd"
bins = "1_33"
# bins = "pi/2_333"

if bins == "1_33":
    file_name = "/test_mi_"
elif bins == "pi/2_333":
    file_name = "/test_mi_new_bins_"
else:
    raise Exception

def rand_jitter(arr):
    stdev = .01*(max(arr)-min(arr))
    return arr + np.random.randn(len(arr)) * stdev

def jitter(x, y, s=20, c='b', marker='o', cmap=None, norm=None, vmin=None, vmax=None, alpha=None, linewidths=None, verts=None, hold=None, **kwargs):
    return plt.scatter(rand_jitter(x), rand_jitter(y), s=s, c=c, marker=marker, cmap=cmap, norm=norm, vmin=vmin, vmax=vmax, alpha=alpha, linewidths=linewidths, verts=verts, hold=hold, **kwargs)

for j in range(10):
    data = pd.ExcelFile(directory + file_name + str(j*frequency) + ".xlsx")
    plt.subplot(5, 2, j + 1)
    plt.xlim((0, 12))
    plt.ylim((-0.1, 1.1))
    if j < 2 or True:
        plt.title("Epoch " + str(j*frequency))
    if j % 2 == 0 or True:
        plt.ylabel("I(Y,T)")
    if j > 7:
        plt.xlabel("I(X,T)")
    # plt.axis("equal")
    # plt.gca().set_aspect('equal', adjustable='box')
    for i, sheetname in enumerate(data.sheet_names):
        if sheetname == mi_estimation_method:
            data_frame = np.array(data.parse(sheetname=sheetname)).T
            print(i, sheetname)
            print(data_frame.T)
            break
    plt.scatter(data_frame[0], data_frame[1], c=np.linspace(0, 1, 8))
plt.show()

# weight decay (L2 regularisation)
# Two functions with some shared and some unique information
# how weights change?
# Partial information theory
# Noisy copy of label
# Just with more epochs (1)
# moitor gradients and weights (2)
# normalised gradients
# Two functions (4) or one function and calculate partial stuff yourself
# Noisy copy (5)

# information decomposition (4)
# Divide 12 inputs to 2 xors, that always have the same bit (fewer  inputs).

# analog to digital, relu.

# check weights when learning XOR. Prove something.

# Check what if happens if you sample 4096 samples randomly etc.
# Its not the number of samples but probably probability distribution.

# Check if 0.5 makes sense.
# p - true label
# q - output

# L(x, N) = p(x) log (1/q(x)) + (1-p(x)) log (1/(1-q(x)))
# N - network?

# L'(x,N) = |round(g(x))-p(x)|
#

# Activation larger range, if all the samples.

# what is the minimum number of samples you need to approximate the function.
# Or how to sample to get better results.

# the amount of outliers is large.

# change the function to sum of variables.

# See what happes without outliers and with outliers (from one side or the other).

# possible problems: more outliers,

# stable distributions.

# increasing the number of inputs.

# information distribution in the input - how it affects the compression. How it affects the ability to learn.
# Dimensionality is also interesting.

# take a function that does not have synergistic information and see if it does the same.
# Outliers more general. Then it should not matter how much synergistic etc information there is.

# sample 3000 samples, but differently (for example, taking a random number between 0 and 12 and this will be the number of ones)
# or from 0 to 24 (20, 17, etc) and mod 12.
# or from all the vectors that have certain number of ones take them a certain number.

# Concentration of measure - sampling randomly from a function

# Dirk - if you go up with n, it will become even more difficult to learn XOR.
# How things scale. Outliers become exponentially more rare.

# Checkerboard pattern. Tradeoff between larger range and pattern squeezed together.

# if small number of variance has a lot of influence, then the fourier transform is concentrated on small sets.
# if large number, then more flat.
# Sample a function that has a certain influence?

# majority function - everybody has least influence.

# 5 boxes of input which we XOR, each box has majority rule inside it. Parameter of influence - ratio between XOR and majority.
# manifold - fold. ones are points in a space. Each layer applies a transformation.

# Only non-negative weights?

# Parity with one (maybe two) hidden layers, with (n log n) nodes.

# training a network gives you a random local minimum.

# Create objective, like linear programming task.

# How things scale - control everything change one thing.

# If information is spread, it is harder for the network to learn (maybe needs more hidden layers). Concentrated - good. Folding is easier.

# Add like 2 nodes to each layer, Dirks conjecture is that then it would work with all 4096 inputs.
# how to we then test the function on unseen data?

# 3000 for 12. What is the 3000 for larger values?
# Find the number of inputs for which the "normal distribution" width is 13.

# Using larger neural network to find a
# Trying larger XORs to see how the learnibility scales? Outscale the input so that the
# Making it larger to be able to put in all the 4096 samples and it would learn it.

# batch normalisation - multiplying (weight?) with a factor

#first try with

# Compare gradients
# Batch size vs

# Automated theorem checking
# Dataset of proofs (low level steps)
# Predicting sequence elements

# Force some weights to be the same (constrained). Translational invariance
# Scaling invariance similarly
# Rotation invariance

# neural network translate to finite state automata

# fixed parameter tracing? kernelazation
# local algorithm - navigating in solution space
# navigate in problem space while preserving solution

# learn Gaussian elimination. Reinforcement learning, measure of diagonality as reward
# Actions - can be continuous.
# Navigate to problems that preserve the solutions
# 5-coloring?

# input = state, output is quality of different actions.
# Markov decision process.


from collections import deque

tunnel = {
    1: ((0,-1),(0,1),(1,0), (-1,0)),
    2: ((-1,0),(1,0)),
    3: ((0,-1),(0,1)),
    4: ((-1,0),(0,1)),
    5: ((0,1), (1,0)),
    6: ((0,-1), (1,0)),
    7: ((0,-1), (-1,0))
}

def BFS(r, c, l):
    global N, M
    queue = deque([(r,c,maps[r][c], 1)])
    maps[r][c] = 0
    cnt = 1
    while queue:
        n = queue.popleft()
        if n[3] < l:
            for d in tunnel[n[2]]:
                nx, ny = n[0]+d[0], n[1]+d[1]
                if 0<=nx<N and 0<=ny<M and maps[nx][ny] and (-d[0], -d[1]) in tunnel[maps[nx][ny]]:
                    queue.append((nx, ny, maps[nx][ny], n[3]+1))
                    maps[nx][ny] = 0
                    cnt += 1
    return cnt

for tc in range(1, int(input())+1):
    # 세로 크기 N, 가로 크기 M, 맨홀 뚜껑이 위치한장소의 세로 위치 R, 가로 위치 C, 그리고 탈출 후 소요된 시간 L
    N, M, R, C, L = map(int, input().split())
    # 그 다음 N 줄에는 지하 터널 지도 정보가 주어지는데, 각 줄마다 M 개의 숫자가 주어진다.
    maps = [list(map(int, input().split())) for _ in range(N)]
    print(f'#{tc}', BFS(R, C, L))


'''
Created on Jan 12, 2019

@author: Mukesh Kumar Singh
'''
import numpy as np
import cv2
import os, glob
import matplotlib.pyplot as plt

def showImages(images, cmap=None):
    cols = 2
    rows = (len(images) + 1) // cols

    plt.figure(figsize=(10, 11))
    for i, image in enumerate(images):
        plt.subplot(rows, cols, i + 1)
        # use gray scale color map if there is only one channel
        cmap = 'gray' if len(image.shape) == 2 else cmap
        plt.imshow(image, cmap=cmap)
        plt.xticks([])
        plt.yticks([])
    plt.tight_layout(pad=0, h_pad=0, w_pad=0)
    plt.show()

def make_line_points(y1, y2, line):
    """
    Convert a line represented in slope and intercept into pixel points
    """
    if line is None:
        return None

    slope, intercept = line

    # make sure everything is integer as cv2.line requires it
    x1 = int((y1 - intercept) / slope)
    x2 = int((y2 - intercept) / slope)
    y1 = int(y1)
    y2 = int(y2)

    return np.array([x1, y1, x2, y2])

def avgSlope(image, lines):
    leftLine = []
    left_weights = []  # (length,)
    rightLine = []
    right_weights = []  # (length,)

    for line in lines:
        x1, y1, x2, y2 = line.reshape(4)
        if (x2 - x1)!=0:
            slope = (y2 - y1) / (x2 - x1)
            intercept = y1 - slope * x1
            length = np.sqrt((y2 - y1) ** 2 + (x2 - x1) ** 2)
            if slope < 0:
                leftLine.append((slope, intercept))
                left_weights.append((length))
            else:
                rightLine.append((slope, intercept))
                right_weights.append((length))

    left_lane = np.dot(left_weights, leftLine) / np.sum(left_weights)  if len(left_weights) > 0 else None
    right_lane = np.dot(right_weights, rightLine) / np.sum(right_weights) if len(right_weights) > 0 else None
    y1 = image.shape[0]
    y2 = y1 * 0.6
    left_line = make_line_points(y1, y2, left_lane)
    right_line = make_line_points(y1, y2, right_lane)
    return np.array([left_line, right_line])

def cannyAlgo(image):
    gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
    blur = cv2.GaussianBlur(gray, (5, 5), 0)
    canny = cv2.Canny(blur, 50, 150)
    return canny

def displayLines(image, lines):
    lineImage1 = np.zeros_like(image)
    if lines is None:
        print(lines)
    else:
        try:
            for line in lines:
                x1, y1, x2, y2 = line.reshape(4)
                cv2.line(lineImage1, (x1, y1), (x2, y2), (255, 0, 255), 10)
        except AttributeError:
            print("shape not found")
    return lineImage1

def regionSelectInLane(image):
    imshape = image.shape
    #poly = np.array([[(20, imshape[0]), (450, 310),(490, 310), (imshape[1], imshape[0])]], dtype=np.int32)
    poly = np.array([[(100, imshape[0]), (450, 310),(490, 310), (imshape[1], imshape[0])]], dtype=np.int32)
    mask = np.zeros_like(image)
    cv2.fillPoly(mask, poly, 255)
    maskedImage = cv2.bitwise_and(image, mask)
    return maskedImage

def drawLaneOnImage(image):
    laneImage = np.copy(image)
    canyImage = cannyAlgo(laneImage)
    cropedImage = regionSelectInLane(canyImage)
    lines = cv2.HoughLinesP(cropedImage, 2, np.pi / 180, 15, np.array([]), minLineLength=5, maxLineGap=2)
    print(lines)
    try:
        avgLines = avgSlope(laneImage, lines)
        lineImage = displayLines(laneImage, avgLines)
        comboImage = cv2.addWeighted(image, 0.8, lineImage, 1, 0)
    except AttributeError:
            print("shape not found")
    return comboImage

lane_images = []
test_images = [cv2.imread(path) for path in glob.glob('test_images/*.jpg')]

for image in test_images:
    lane_images.append(drawLaneOnImage(image))
showImages(lane_images)

#cap = cv2.VideoCapture("test_videos/solidWhiteRight.mp4")
# cap=cv2.VideoCapture("test_videos/solidYellowLeft.mp4")
cap = cv2.VideoCapture("test_videos/challenge.mp4")
while(cap.isOpened()):
    _, color_frame = cap.read()
    cv2.imshow("result", drawLaneOnImage(color_frame))
    cv2.waitKey(50)


import setuptools

# Reads the content of your README.md into a variable to be used in the setup below
def readme():
    with open("README.md", encoding='utf-8') as f:
        README = f.read()
    return README

setuptools.setup(
    name='classification_transformers',                           # should match the package folder
    packages=['classification_transformers'],                     # should match the package folder
    version='0.0.1',                                # important for updates
    #license='MIT',                                  # should match your chosen license
    description='Use the package for easy configuration of the huggingface models',
    long_description=readme(),              # loads your README.md
    long_description_content_type="text/markdown",  # README.md is of type 'markdown'
    author='Harshad Patil',
    author_email='hhpatil001@gmail.com',
    url='https://github.com/harshad317/custom_transformers',
    #project_urls = {                                # Optional
    #    "Bug Tracker": "https://github.com/mike-huls/toolbox_public/issues"
    #},
    install_requires=['requests'],                  # list all packages that your package uses
    keywords=["pypi", "mikes_toolbox", "tutorial"], #descriptive meta-data
    classifiers=[                                   # https://pypi.org/classifiers
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python :: 3.7',
        'Programming Language :: Python :: 3.9',
        'Programming Language :: Python :: 3.10',
    ],

)


#imports
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import itertools
import collections
import nltk
from nltk import trigrams
from nltk.corpus import stopwords
import re
import networkx as nx
import warnings
import sqlalchemy
import string
import math
from sqlalchemy import create_engine
from tokenizer import *
import json
from matplotlib.colors import rgb2hex

#seaborn settings
sns.set(font_scale=1.5)
sns.set_style("whitegrid")

#allow use of arguments
import argparse
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument("--type", help="Specify whether to use posts or comments. Default is comments.", choices=['posts','comments'], default='comments')
arg_parser.add_argument("--pages", help="Specify whether to use all facebook pages, only US facebook pages, or only Guyana faebook pages. Default is us", choices=['all','us','guy'], default='us')
arg_parser.add_argument("--date", help="Earliest date for posts/comments in format YYYY-MM-DD. Default is 2019-04-01.", default="2019-04-01")
arg_parser.add_argument("--num_trigrams", help="Number of trigrams to show in barchart and graph. Default is 30.", type=int, default=30)
arg_parser.add_argument("--ignore", help="Ignore warnings", action="store_true")
args = arg_parser.parse_args()

#read off input values
if args.ignore:
    print('Ignoring all warnings...')
    warnings.filterwarnings("ignore", category=DeprecationWarning)
print("Using %s for analysis" % args.type)
print("Using start_date of: ",args.date)

#stop words for both types of pages
us_stop_words = []
guy_stop_words = []

#stop lemmas
us_stop_lemmas = ["en","la","de","desde","los","eso","es"]

#define and instantiate tokenizers
us_tokenizer = Tokenizer(stop_words=us_stop_words, case_sensitive=False, stop_lemmas=us_stop_lemmas, lemma_token=False, lower_token=True)
guy_tokenizer = Tokenizer(stop_words=guy_stop_words, case_sensitive=False, lemma_token=False, lower_token=True)

#pages used for facebook pull
if args.pages == 'all':
    page_ids = ['AFSOUTHNewHorizons','USEmbassyGeorgetown','southcom','dpiguyana','AFSouthern','NewsSourceGuyana','655452691211411','kaieteurnewsonline','demwaves','CapitolNewsGY','PrimeNewsGuyana','INews.Guyana','stabroeknews','NCNGuyanaNews','dailynewsguyana','actionnewsguyana','gychronicle','gytimes','newsroomgy']
elif args.pages == 'us':
    page_ids = ['AFSOUTHNewHorizons','USEmbassyGeorgetown','southcom','AFSouthern']
elif args.pages == 'guy':
    page_ids = ['dpiguyana','NewsSourceGuyana','655452691211411','kaieteurnewsonline','demwaves','CapitolNewsGY','PrimeNewsGuyana','INews.Guyana','stabroeknews','NCNGuyanaNews','dailynewsguyana','actionnewsguyana','gychronicle','gytimes','newsroomgy']

#lookup for detemining correct tokenizer
page_to_tokenizer = {
    'AFSOUTHNewHorizons': us_tokenizer,
    'USEmbassyGeorgetown': us_tokenizer,
    'southcom': us_tokenizer,
    'dpiguyana': guy_tokenizer,
    'AFSouthern': us_tokenizer,
    'NewsSourceGuyana': guy_tokenizer,
    '655452691211411': guy_tokenizer,
    'kaieteurnewsonline': guy_tokenizer,
    'demwaves': guy_tokenizer,
    'CapitolNewsGY': guy_tokenizer,
    'PrimeNewsGuyana': guy_tokenizer,
    'INews.Guyana': guy_tokenizer,
    'stabroeknews': guy_tokenizer,
    'NCNGuyanaNews': guy_tokenizer,
    'dailynewsguyana': guy_tokenizer,
    'actionnewsguyana': guy_tokenizer,
    'gychronicle': guy_tokenizer,
    'gytimes': guy_tokenizer,
    'newsroomgy': guy_tokenizer
}

#connect to database
engine = create_engine('sqlite:///./nh19_fb.db')

#get all comments for this year
raw_df = pd.read_sql("""select * from {0} where created_time > '{1}'""".format(args.type, args.date),engine)

#find date of most recent_comment
relevant_pages = raw_df[raw_df['page'].isin(page_ids)]
most_recent_date = relevant_pages.created_time.max()[:10]

#filter for page and store list of comments in
page_comments = {}
for pageid in page_ids:
    tmp_df = raw_df[raw_df['page'] == pageid]
    page_comments[pageid] = tmp_df[tmp_df['message'].notnull()].message.to_list()

#tokenize, create trigram counts, only keep most common trigrams and trigrams that show up at least a certain amount of times
page_trigram_dict = {}
page_tokens = {}
for page in page_comments:
    print('Tokenizing {0} {1} from {2}'.format(len(page_comments[page]), args.type, page))
    tokens = page_to_tokenizer[page].tokenize(page_comments[page], return_docs=False)
    page_tokens[page] = tokens
    trigrams = list(nltk.trigrams(tokens))
    #keep top n trigrams
    trigram_count = collections.Counter(trigrams).most_common(args.num_trigrams)
    #make dictionary with trigram as key and count as value; only keep trigrams that appear more than once
    trigram_dict = {item[0]: item[1] for item in trigram_count if item[1] > 1}
    page_trigram_dict[page] = trigram_dict

#get trigram counts for overall chart
all_tokens = [token for page in page_tokens for token in page_tokens[page]]
all_trigrams = list(nltk.trigrams(all_tokens))
all_trigram_counts = collections.Counter(all_trigrams).most_common(args.num_trigrams)
all_trigram_dict = {item[0]: item[1] for item in all_trigram_counts if item[1] > 1}

#barchart for most common trigrams
for i,page in enumerate(page_trigram_dict):
    plt.figure(i)
    keys = [trigram[0] + '_' + trigram[1] + '_' + trigram[2] for trigram in page_trigram_dict[page].keys()]
    plt.bar(keys,list(page_trigram_dict[page].values()))
    plt.xticks(rotation=40, ha='right')
    plt.subplots_adjust(bottom=0.4)
    plt.xlabel("Trigram")
    plt.ylabel("Frequency")
    ax = plt.gca()
    ax.tick_params(axis='both', which='major', labelsize=12)
    plt.title('{0} Trigrams, {1} - {2}'.format(page, args.date, most_recent_date), fontsize = 20)
    plt.savefig('./trigramCharts/{0}_{1}_trigramBar_{2}_to_{3}.png'.format(page,args.type,args.date,most_recent_date))

#barchart for most common trigrams overall
plt.figure(len(page_trigram_dict))
all_keys = [trigram[0] + '_' + trigram[1] + '_' + trigram[2] for trigram in all_trigram_dict.keys()]
plt.bar(all_keys,list(all_trigram_dict.values()))
plt.xticks(rotation=40, ha='right')
plt.subplots_adjust(bottom=0.4)
plt.xlabel("Trigram")
plt.ylabel("Frequency")
ax = plt.gca()
ax.tick_params(axis='both', which='major', labelsize=12)
plt.title('Overall {0} Trigrams, {1} - {2}'.format(args.pages.upper(), args.date, most_recent_date), fontsize = 20)
plt.savefig('./trigramCharts/overall_{0}_{1}_trigramBar_{2}_to_{3}.png'.format(args.pages,args.type,args.date,most_recent_date))

#set scaling factor for how spread out trigrams are (smaller makes more spread)
scale_factor = 5
weight_mult = args.num_trigrams/scale_factor

#Graphing network digram
for page in page_trigram_dict:
    G = nx.Graph()
    node_dict = {}

    #add all edges (trigrams) to graph
    for k,v in page_trigram_dict[page].items():
        G.add_edge(k[0],k[1], weight=(v))
        G.add_edge(k[1],k[2], weight=(v))
        #populate word frequency from trigrams alone
        for word in k:
            if word in node_dict.keys():
                node_dict[word] += v
            else:
                node_dict[word] = v

    max_weight = max([v for v in page_trigram_dict[page].values()])
    fig,ax = plt.subplots(figsize=(18,14))
    pos = nx.spring_layout(G,k=max(math.log(max_weight+1,10),1))

    #draw graph
    nx.draw_networkx(G,pos,font_size=16,width=3,edge_color='grey',node_size=200,node_color=list(node_dict.values()),cmap=plt.cm.plasma,with_labels=False,ax=ax)
    plt.title('{0} Trigram Diagram, {1} - {2}'.format(page, args.date, most_recent_date))

    #assign color to each node based on node weight (number of bigram connections to each node)
    color_scale = plt.cm.plasma.__copy__()
    max_val = max(node_dict.values())
    node_colors = {node: rgb2hex(color_scale(node_dict[node]/max_val)) for node in node_dict}

    #get nodes and edges
    cyto_dict = nx.readwrite.json_graph.cytoscape_data(G)['elements']
    #add weight and color to each node
    for node in cyto_dict['nodes']:
        node['data']['weight'] = node_dict[node['data']['id']]
        node['data']['node_color'] = node_colors[node['data']['id']]
    #write json file to disk
    cyto_json_path = "./trigramCharts/cyto_tri_json_{0}_{1}_{2}_to_{3}.json".format(page, args.type, args.date, most_recent_date)
    with open(cyto_json_path,"w") as cyto_file:
        cyto_file.write(json.dumps(cyto_dict))

    #turn off grid lines and both axes
    ax.axis('off')

    #labelling diagram
    for key, value in pos.items():
        x = value[0]
        y = value[1]+0.05
        ax.text(x,y,s=key,bbox=dict(facecolor='grey',edgecolor='black',alpha=0.1),horizontalalignment='center',fontsize=13)

    #create colorbar as legend for colormap
    sm = plt.cm.ScalarMappable(cmap=plt.cm.plasma)
    sm._A = list(node_dict.values())
    cbar = plt.colorbar(sm)
    cbar.ax.get_yaxis().labelpad = 15
    cbar.ax.set_ylabel('Word Frequency', rotation=270)
    #save figure
    plt.savefig('./trigramCharts/{0}_{1}_trigramGraph_{2}_to_{3}.png'.format(page,args.type,args.date,most_recent_date))

#Graphing for overall plot
G = nx.Graph()
node_dict = {}

for k,v in all_trigram_dict.items():
    G.add_edge(k[0],k[1], weight=(v))
    G.add_edge(k[1],k[2], weight=(v))
    #populate word frequency from trigrams alone
    for word in k:
        if word in node_dict.keys():
            node_dict[word] += v
        else:
            node_dict[word] = v

max_weight = max([v for v in all_trigram_dict.values()])
fig,ax = plt.subplots(figsize=(18,14))
pos = nx.spring_layout(G,k=max(math.log(max_weight+1,10),1))

#draw graph
nx.draw_networkx(G,pos,font_size=16,width=3,node_size=200,node_color=list(node_dict.values()),edge_color='grey',cmap=plt.cm.plasma, with_labels=False,ax=ax)
plt.title('Overall {0} Trigram Diagram, {1} - {2}'.format(args.pages.upper(), args.date, most_recent_date))

color_scale = plt.cm.plasma.__copy__()
max_val = max(node_dict.values())
node_colors = {node: rgb2hex(color_scale(node_dict[node]/max_val)) for node in node_dict}

cyto_dict = nx.readwrite.json_graph.cytoscape_data(G)['elements']
for node in cyto_dict['nodes']:
    node['data']['weight'] = node_dict[node['data']['id']]
    node['data']['node_color'] = node_colors[node['data']['id']]

cyto_json_path = "./trigramCharts/cyto_tri_json_{0}_{1}_{2}_to_{3}.json".format(args.pages, args.type, args.date, most_recent_date)
with open(cyto_json_path,"w") as cyto_file:
    cyto_file.write(json.dumps(cyto_dict))

#turn off grid lines and both axes
ax.axis('off')

#labelling diagram
for key, value in pos.items():
    x = value[0]
    y = value[1] + 0.05
    ax.text(x,y,s=key,horizontalalignment='center',fontsize=13,bbox=dict(facecolor='grey',edgecolor='black',alpha=0.1))

#create colorbar as legend for colormap
sm = plt.cm.ScalarMappable(cmap=plt.cm.plasma)
sm._A = list(node_dict.values())
cbar = plt.colorbar(sm)
cbar.ax.get_yaxis().labelpad = 15
cbar.ax.set_ylabel('Word Frequency', rotation=270)
#save figure
plt.savefig('./trigramCharts/overall_{0}_{1}_trigramGraph_{2}_to_{3}.png'.format(args.pages,args.type,args.date,most_recent_date))

plt.show()


from django import forms
from .models import SellStock, BuyStock, AllStocks

class DateInput(forms.DateInput):
    input_type = 'date'

class AllStockForm(forms.ModelForm):
    class Meta:
        model = AllStocks
        fields = ('stock_name', 'currency')

class BuyForm(forms.ModelForm):
    class Meta:
        model = BuyStock
        widgets = {'stock_buy_transacted_date': DateInput()}
        fields = ('stock_buy_transacted_date',
                  'stock_buy_units', 'stock_price_per_unit',
                  'fee')

class SellForm(forms.ModelForm):
    class Meta:
        model = SellStock
        widgets = {'stock_sell_transacted_date': DateInput()}
        fields = ("stock_sell_transacted_date", "stock_sell_units"
                  , "stock_price_per_unit", "fee")

from django.contrib import admin

# Register your models here.


import os
import sys
sys.path.append("..")

import numpy as np

import json

A2D2_LOCATION = '/mnt/ml-data-storage/jens/A2D2/'

DATA_DIR = '/mnt/ml-data-storage/jens/A2D2/'
IMAGE_DIRS = os.path.join(DATA_DIR, 'camera_lidar_semantic')

from geopy.geocoders import Nominatim
from IPython.display import clear_output
import time

geolocator = Nominatim(user_agent='A2D2-data')

runs = os.listdir(DATA_DIR)
for r in runs:
    if os.path.isdir(os.path.join(DATA_DIR,r)) and r != 'camera_lidar_semantic':
        bus_file = os.path.join(DATA_DIR, r, 'bus', r.replace('_', '')) + '_bus_signals.json'
        with open(bus_file) as f:
            data = json.load(f)
            for i in range(len(data)):
                if 'latitude_degree' in data[i]['flexray'].keys():
                    lat, long = data[i]['flexray']['latitude_degree']['values'], data[i]['flexray']['longitude_degree']['values']
                    location = geolocator.reverse(str(lat[0])+ ","+str(long[0]))
                    try:
                        city = location.raw['address']
                        pos = city['city']
                    except:
                        try:
                            pos = city['village']
                        except:
                            #print(location.raw['address'])
                            continue
                            #try:
                            #    pos = city['town']
                            #except:
                            #    continue
                    print(r, pos, "postcode:", location.raw['address']['postcode'])
                    break


from typing import Dict, List, Optional
import weakref

from tensorflow.core.protobuf import trackable_object_graph_pb2

from tensorflow.dtensor.python import api
from tensorflow.dtensor.python import d_variable
from tensorflow.dtensor.python import gen_dtensor_ops
from tensorflow.dtensor.python import layout
from tensorflow.dtensor.python import save_restore
from tensorflow.python.checkpoint import checkpoint as util
from tensorflow.python.checkpoint import checkpoint_options
from tensorflow.python.checkpoint import graph_view as graph_view_lib
from tensorflow.python.checkpoint import restore as restore_lib
from tensorflow.python.eager import context
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import errors_impl
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.trackable import base
from tensorflow.python.trackable import data_structures
from tensorflow.python.training import py_checkpoint_reader
from tensorflow.python.training.saving import saveable_object
from tensorflow.python.training.saving import saveable_object_util
from tensorflow.python.util import deprecation
from tensorflow.python.util import nest
from tensorflow.python.util.tf_export import tf_export

class _DSaver:  # pylint: disable=protected-access
  """A single device saver that places tensors on DTensor Device."""

  def __init__(self, mesh: layout.Mesh,
               saveable_objects: List[saveable_object.SaveableObject]):
    self._saveable_objects = saveable_objects
    self._mesh = mesh

  def save(
      self,
      file_prefix: str,
      options: Optional[checkpoint_options.CheckpointOptions] = None
  ) -> Optional[ops.Operation]:
    """Saves the saveable objects to a checkpoint with `file_prefix`.

    Also query the generated shards from the distributed DTensor SaveV2 ops and
    do a MergeV2 on those. Each op here is backed by a global_barrier to avoid
    racing from multiple clients.

    Args:
      file_prefix: A string or scalar string Tensor containing the prefix to
        save under.
      options: Optional `CheckpointOptions` object. This is unused in DTensor.

    Returns:
      An `Operation`, or None when executing eagerly.
    """
    if options is not None and options.experimental_io_device is not None:
      raise ValueError(
          "Specified experimental_io_device in DTensor checkpoint is not supported."
      )
    del options
    tensor_names = []
    tensors = []
    tensor_slices = []
    for saveable in self._saveable_objects:
      for spec in saveable.specs:
        tensor = spec.tensor
        # A tensor value of `None` indicates that this SaveableObject gets
        # recorded in the object graph, but that no value is saved in the
        # checkpoint.
        if tensor is not None:
          if api.device_name() != spec.device:
            # Some small tensors are placed on CPU0 from save manager and
            # broadcasted to DTensor mesh, e,g., SaveCounter.
            tensor = api.pack([tensor] *
                              self._mesh.host_mesh().num_local_devices(),
                              layout.Layout.replicated(
                                  self._mesh.host_mesh(),
                                  rank=tensor.shape.rank))
          tensor_names.append(spec.name)
          tensors.append(tensor)
          tensor_slices.append(spec.slice_spec)
    return save_restore.sharded_save(self._mesh, file_prefix, tensor_names,
                                     tensor_slices, tensors)

  def restore(
      self,
      file_prefix: str,
      options: Optional[checkpoint_options.CheckpointOptions] = None
  ) -> Dict[str, ops.Operation]:
    """Restore the saveable objects from a checkpoint with `file_prefix`.

    Args:
      file_prefix: A string or scalar string Tensor containing the prefix for
        files to read from.
      options: Optional `CheckpointOptions` object. This is unused in DTensor.

    Returns:
      A dictionary mapping from SaveableObject names to restore operations.
    """
    if options is not None and options.experimental_io_device is not None:
      raise ValueError(
          "Specified experimental_io_device in DTensor checkpoint is not "
          "supported.")
    del options
    restore_specs = []
    tensor_structure = []
    for saveable in self._saveable_objects:
      saveable_tensor_structure = []
      tensor_structure.append(saveable_tensor_structure)
      # DTensor change 1 : Gather shapes and layout from original saveable
      # specs.
      # Note that this relies on the fact that the variables are already
      # initialized -- which isn't the behavior we want eventually.
      # TODO(b/159035705): Handle the variable initialization in restore.
      for spec in saveable.specs:
        saveable_tensor_structure.append(spec.name)
        if isinstance(spec, d_variable.DSaveSpec):
          restore_specs.append((spec.name, spec.slice_spec, spec.dtype,
                                spec.layout, spec.global_shape))
        # Fall back to replicated layouts for non-DTensor saves that constructs
        # normal SaveSpec.
        elif isinstance(spec, saveable_object.SaveSpec):
          restore_specs.append(
              (spec.name, spec.slice_spec, spec.dtype,
               layout.Layout.replicated(self._mesh.host_mesh(),
                                        spec.tensor.shape.rank).to_string(),
               spec.tensor.shape.as_list()))
    tensor_names, tensor_slices, tensor_dtypes, layouts, global_shapes = zip(
        *restore_specs)
    with ops.device(api.device_name()):
      # DTensor change 2 : Run on customized DTensor RestoreV2 op rather than
      # stock TF io_ops.RestoreV2.
      restored_tensors = gen_dtensor_ops.d_tensor_restore_v2(
          prefix=file_prefix,
          tensor_names=tensor_names,
          shape_and_slices=tensor_slices,
          input_shapes=global_shapes,
          input_layouts=layouts,
          dtypes=tensor_dtypes)
    structured_restored_tensors = nest.pack_sequence_as(tensor_structure,
                                                        restored_tensors)
    restore_ops = {}
    for saveable, restored_tensors in zip(self._saveable_objects,
                                          structured_restored_tensors):
      restore_ops[saveable.name] = saveable.restore(
          restored_tensors, restored_shapes=None)
    return restore_ops

class _DCheckpointRestoreCoordinator(util._CheckpointRestoreCoordinator):  # pylint: disable=protected-access
  """Holds the status of an object-based checkpoint load."""

  def __init__(self, mesh: layout.Mesh, **kwargs):
    super().__init__(**kwargs)
    self._mesh = mesh

  def restore_saveables(self,
                        tensor_saveables: Dict[str,
                                               saveable_object.SaveableObject],
                        python_positions: List[restore_lib.CheckpointPosition],
                        registered_savers: Optional[Dict[str, Dict[
                            str, base.Trackable]]] = None,
                        reader: py_checkpoint_reader.NewCheckpointReader = None
                        ) -> Optional[List[ops.Operation]]:
    """Run or build restore operations for SaveableObjects.

    Args:
      tensor_saveables: `SaveableObject`s which correspond to Tensors.
      python_positions: `CheckpointPosition`s which correspond to `PythonState`
        Trackables bound to the checkpoint.
      registered_savers: a dict mapping saver names-> object name -> Trackable.
        This argument is not implemented for DTensorCheckpoint.
      reader: A CheckpointReader. Creates one lazily if None.

    Returns:
      When graph building, a list of restore operations, either cached or newly
      created, to restore `tensor_saveables`.
    """
    del registered_savers

    restore_ops = []
    # Eagerly run restorations for Python state.
    if python_positions:
      # Lazily create the NewCheckpointReader, since this requires file access
      # and we may not have any Python saveables.
      if reader is None:
        reader = py_checkpoint_reader.NewCheckpointReader(self.save_path_string)
      for position in python_positions:
        key = position.object_proto.attributes[0].checkpoint_key
        position.trackable.deserialize(reader.get_tensor(key))

    # If we have new SaveableObjects, extract and cache restore ops.
    if tensor_saveables:
      validated_saveables = saveable_object_util.validate_and_slice_inputs(
          tensor_saveables)
      validated_names = set(saveable.name for saveable in validated_saveables)
      if set(tensor_saveables.keys()) != validated_names:
        raise AssertionError(
            ("Saveable keys changed when validating. Got back %s, was "
             "expecting %s") % (tensor_saveables.keys(), validated_names))
      # DTensor change: Use _DSaver that does restore on DTensor with
      # customized DTensorRestoreV2 op.
      new_restore_ops = _DSaver(self._mesh, validated_saveables).restore(
          self.save_path_tensor, self.options)
      if not context.executing_eagerly():
        for name, restore_op in sorted(new_restore_ops.items()):
          restore_ops.append(restore_op)
          assert name not in self.restore_ops_by_name
          self.restore_ops_by_name[name] = restore_op
    return restore_ops

class DTrackableSaver(util.TrackableSaver):
  """A DTensor trackable saver that uses _SingleDeviceSaver."""

  def __init__(self, mesh: layout.Mesh, graph_view):
    super(DTrackableSaver, self).__init__(graph_view)
    self._mesh = mesh

  def _gather_saveables(self, object_graph_tensor=None):
    # Since the base Checkpoint class does not return SaveableObjects, re-use
    # the saveables cache or generate new Saveables.
    (serialized_tensors, feed_additions, registered_savers,
     graph_proto) = self._gather_serialized_tensors(object_graph_tensor)

    saveables_dict = self._saveables_cache
    if saveables_dict is None:
      # Get and remove object graph tensor from `serialized_tensors`, because
      # the function `serialized_tensors_to_saveable_cache` isn't equipped
      # to handle it.
      object_graph_tensor = serialized_tensors.pop(
          None)[base.OBJECT_GRAPH_PROTO_KEY]
      saveables_dict = (
          saveable_object_util.serialized_tensors_to_saveable_cache(
              serialized_tensors))
    named_saveable_objects = []
    for saveable_by_name in saveables_dict.values():
      for saveables in saveable_by_name.values():
        named_saveable_objects.extend(saveables)
    named_saveable_objects.append(
        base.NoRestoreSaveable(
            tensor=object_graph_tensor,
            name=base.OBJECT_GRAPH_PROTO_KEY))
    return (named_saveable_objects, graph_proto, feed_additions,
            registered_savers)

  def _save_cached_when_graph_building(self,
                                       file_prefix,
                                       object_graph_tensor,
                                       options,
                                       update_ckpt_state=False):
    """Create or retrieve save ops, overrides parents's private method.

    Args:
      file_prefix: The prefix for saved checkpoint files.
      object_graph_tensor: A `Tensor` to which the current object graph will be
        fed.
      options: `CheckpointOptions` object.
      update_ckpt_state: Optional bool flag. Indiciate whether the internal
        checkpoint state needs to be updated. This is used for async checkpoint,
        which DTrackableSaver currently does not support.
    TODO(chienchunh): Implement async checkpoint for DTrackableSaver.

    Returns:
      A two-element tuple with a filename tensor and a feed_dict of tensors to
      feed when running it (if graph building). The feed dict contains the
      current object graph and any Python state to be saved in the
      checkpoint. When executing eagerly only the first argument is meaningful.
    """
    (named_saveable_objects, graph_proto, feed_additions,
     unused_registered_savers) = self._gather_saveables(
         object_graph_tensor=object_graph_tensor)
    if (self._last_save_object_graph != graph_proto
        # When executing eagerly, we need to re-create SaveableObjects each time
        # save() is called so they pick up new Tensors passed to their
        # constructors. That means the Saver needs to be copied with a new
        # var_list.
        or context.executing_eagerly() or ops.inside_function()):
      # This is needed to avoid MultiDeviceSaver creating unnecessary MergeV2
      # ops in DTensor. It is an issue when saving TPU Variables on host CPU
      # mesh given our limited expressiveness in API and hard-coded logic in
      # broadcasting -- for a small constant Tensor with no extra information,
      # we place it on the first registered mesh(A.K.A. default mesh).
      saver = _DSaver(self._mesh, named_saveable_objects)
      save_op = saver.save(file_prefix, options=options)
      with ops.device("/cpu:0"):
        with ops.control_dependencies([save_op]):
          self._cached_save_operation = array_ops.identity(file_prefix)
      self._last_save_object_graph = graph_proto
    return self._cached_save_operation, feed_additions

  # TODO(b/180466245): Use proper mesh placement semantic.
  def restore(self, save_path, options=None):
    """Restore a training checkpoint with host mesh placement."""
    options = options or checkpoint_options.CheckpointOptions()
    if save_path is None:
      return util.InitializationOnlyStatus(self._graph_view, ops.uid())
    reader = py_checkpoint_reader.NewCheckpointReader(save_path)
    graph_building = not context.executing_eagerly()
    if graph_building:
      dtype_map = None
    else:
      dtype_map = reader.get_variable_to_dtype_map()
    try:
      object_graph_string = reader.get_tensor(base.OBJECT_GRAPH_PROTO_KEY)
    except errors_impl.NotFoundError:
      # The object graph proto does not exist in this checkpoint. Try the
      # name-based compatibility mode.
      restore_coordinator = util._NameBasedRestoreCoordinator(  # pylint: disable=protected-access
          save_path=save_path,
          dtype_map=dtype_map)
      if not graph_building:
        for existing_trackable in self._graph_view.list_objects():
          # pylint: disable=protected-access
          existing_trackable._maybe_initialize_trackable()
          existing_trackable._name_based_restores.add(restore_coordinator)
          existing_trackable._name_based_attribute_restore(restore_coordinator)
          # pylint: enable=protected-access
      return util.NameBasedSaverStatus(
          restore_coordinator, graph_view=self._graph_view)

    if graph_building:
      if self._file_prefix_placeholder is None:
        # DTensor change: provide a hint for mesh broadcasting to put the input
        # onto the host mesh.
        self._file_prefix_placeholder = api.pack(
            [constant_op.constant("model")] * self._mesh.num_local_devices(),
            layout.Layout.replicated(self._mesh.host_mesh(), rank=0))
      file_prefix_tensor = self._file_prefix_placeholder
      file_prefix_feed_dict = {self._file_prefix_placeholder: save_path}
    else:
      # DTensor change: provide a hint for mesh broadcasting to put the input
      # onto the host mesh.
      file_prefix_tensor = api.pack(
          [constant_op.constant(save_path)] * self._mesh.num_local_devices(),
          layout.Layout.replicated(self._mesh.host_mesh(), rank=0))
      file_prefix_feed_dict = None
    object_graph_proto = (trackable_object_graph_pb2.TrackableObjectGraph())
    object_graph_proto.ParseFromString(object_graph_string)
    # DTensor Change: Hook the proper DSaver in restore.
    checkpoint = _DCheckpointRestoreCoordinator(
        mesh=self._mesh,
        object_graph_proto=object_graph_proto,
        save_path=save_path,
        save_path_tensor=file_prefix_tensor,
        reader=reader,
        restore_op_cache=self._restore_op_cache,
        graph_view=self._graph_view,
        options=options,
        saveables_cache=self._saveables_cache)
    restore_lib.CheckpointPosition(
        checkpoint=checkpoint, proto_id=0).restore(self._graph_view.root)

    # Attached dependencies are not attached to the root, so should be restored
    # separately.
    if self._graph_view.attached_dependencies:
      for ref in self._graph_view.attached_dependencies:
        if ref.name == "root":
          # Root dependency is automatically added to attached dependencies --
          # this can be ignored since it maps back to the root object.
          continue
        proto_id = None
        # Find proto ID of attached dependency (if it is in the proto).
        for proto_ref in object_graph_proto.nodes[0].children:
          if proto_ref.local_name == ref.name:
            proto_id = proto_ref.node_id
            break

        if proto_id in checkpoint.object_by_proto_id:
          # Object has already been restored. This can happen when there's an
          # indirect connection from the attached object to the root.
          continue

        restore_lib.CheckpointPosition(
            checkpoint=checkpoint, proto_id=proto_id).restore(ref.ref)

    load_status = util.CheckpointLoadStatus(
        checkpoint,
        graph_view=self._graph_view,
        feed_dict=file_prefix_feed_dict)
    return load_status

@deprecation.deprecated(
    date=None,
    instructions="Please use tf.train.Checkpoint instead of DTensorCheckpoint. "
    "DTensor is integrated with tf.train.Checkpoint and it can be "
    "used out of the box to save and restore dtensors.")
@tf_export("experimental.dtensor.DTensorCheckpoint", v1=[])
class DTensorCheckpoint(util.Checkpoint):
  """Manages saving/restoring trackable values to disk, for DTensor."""

  def __init__(self, mesh: layout.Mesh, root=None, **kwargs):
    super(DTensorCheckpoint, self).__init__(root=root, **kwargs)
    self._mesh = mesh

    saver_root = self
    attached_dependencies = None
    self._save_counter = None  # Created lazily for restore-on-create.
    self._save_assign_op = None

    if root:
      util._assert_trackable(root, "root")
      saver_root = root
      attached_dependencies = []

      # All keyword arguments (including root itself) are set as children
      # of root.
      kwargs["root"] = root
      root._maybe_initialize_trackable()

      self._save_counter = data_structures.NoDependency(
          root._lookup_dependency("save_counter"))
      self._root = data_structures.NoDependency(root)

    for k, v in sorted(kwargs.items(), key=lambda item: item[0]):
      setattr(self, k, v)

      # Call getattr instead of directly using v because setattr converts
      # v to a Trackable data structure when v is a list/dict/tuple.
      converted_v = getattr(self, k)
      util._assert_trackable(converted_v, k)

      if root:
        # Make sure that root doesn't already have dependencies with these names
        attached_dependencies = attached_dependencies or []
        child = root._lookup_dependency(k)
        if child is None:
          attached_dependencies.append(base.TrackableReference(k, converted_v))
        elif child != converted_v:
          raise ValueError(
              "Cannot create a Checkpoint with keyword argument {name} if "
              "root.{name} already exists.".format(name=k))
    # DTensor Change:
    # Override the parents saver with DTrackableSaver with _SingleDeviceSaver.
    self._saver = DTrackableSaver(
        mesh,
        graph_view_lib.ObjectGraphView(
            weakref.ref(saver_root),
            attached_dependencies=attached_dependencies))


#Given an array of integers arr, replace each element with its rank.
#The rank represents how large the element is. The rank has the following rules:
#Rank is an integer starting from 1.
#The larger the element, the larger the rank. If two elements are equal, their rank must be the same.
#Rank should be as small as possible.

class Solution:
    def arrayRankTransform(self, arr: List[int]) -> List[int]:
        ans = []
        d = {}
        k = 1
        sort = sorted(arr)
        for i in range(len(arr)):
            if sort[i] not in d:
                d[sort[i]] = k
                k += 1

        for i in arr:
            ans.append(d[i])
        return ans


# to create inverted index for the given corpus

from Indexer import IndexCreator

stemmedCorpus = open("cacm_stem.txt")

for line in stemmedCorpus:
	tokens = line.split()
	if tokens[0] == "#":
		docName = tokens[1]
		text = ""
		if len(docName) == 1:
			text = "000"
		elif len(docName) == 2:
			text = "00"
		elif len(docName) == 3:
			text = "0"
		else:
			text = ""

		file = open("cacm/CACM-" + text + docName + ".txt", 'w')
	else:
		for token in tokens:
			file.write(token)
			file.write(" ")
rootdir = "cacm/"

index = IndexCreator()
index.docToID(rootdir)
invertedIndex = index.getIndex(rootdir, 1)

# invertedIndex = index.getIndex(rootdir, 2)
# invertedIndex = index.getIndex(rootdir, 3)

# to get the count of tokens in each of the document
tokenCount = index.getTokensInADoc()
tokensDoc = index.getTokens()
# sstopList = index.generateStopList()

# to get the index and the document statistics
index.dumpIndex(invertedIndex,1)
index.storeDocStatistics(invertedIndex,1)

# index.dumpIndex(invertedIndex,2)
# index.storeDocStatistics(invertedIndex,2)

# index.dumpIndex(invertedIndex,3)
# index.storeDocStatistics(invertedIndex,3)


import logging
import numpy
import numpy as np
import pandas as pd
from pandas.api.types import CategoricalDtype
from math import *
from datetime import datetime
from operator import itemgetter
from zipfile import ZipFile
from io import BytesIO
import pickle
from urllib.request import urlopen

from IPython.display import Image
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.express as px

from numpy import inf
from scipy.stats import kurtosis, skew

from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from sklearn import ensemble
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
from sklearn.metrics import roc_auc_score
from sklearn import metrics
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import LinearRegression
from sklearn.metrics import average_precision_score
from sklearn.metrics import mean_squared_error

import warnings
warnings.filterwarnings('ignore')

from fair_func import acfmetrics

class Generic:

    def __init__(self, data, protected_features, independent_features, target_variable, is_fair, is_sensitive=None):
        """function to initialize class"""
        self.data = data
        y=self.data[target_variable]*1000
        x=self.data.drop(columns=[target_variable])
        self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(x, y, test_size=0.20, random_state=3000)
        self.protected_features = protected_features
        self.independent_features = independent_features
        self.target_variable = target_variable
        self.sens_test = self.X_test[self.protected_features] if self.protected_features is not "" else None
        self.counter_sens_test = None
        self.is_fair = is_fair
        self.is_sensitive = is_sensitive
        self.model = None
        self.predictions = None
        self.pred_probs = None

    def get_residuals_train_data(self):
        """function to get residuals for train data
           residual = diff(predicted,actual)"""
        residuals_dict_train = {}
        sens_train=self.X_train[self.protected_features]
        for feature in self.independent_features:
            clf_feature_train = LinearRegression().fit(sens_train, self.X_train[feature])
            residual_train = self.X_train[feature] - clf_feature_train.predict(sens_train)
            residuals_dict_train[f"{feature}R"] = residual_train

        df_R_train = pd.DataFrame(residuals_dict_train)
        return df_R_train

    def fit(self, X_train=None):
        """function to fit model"""
        if X_train is None:
            self.model = LogisticRegression(random_state=0, solver='liblinear', multi_class='ovr').fit(self.X_train, self.y_train)
        else:
            self.model = LogisticRegression(random_state=0, solver='liblinear', multi_class='ovr').fit(X_train, self.y_train)
        return self.model

    def get_residuals_test_data(self,is_cuf=False):
        """function to get residuals for test data
           residual = diff(predicted,actual)"""
        residuals_dict_test = {}
        for feature in self.independent_features:
            clf_feature_test = LinearRegression().fit(self.sens_test, self.X_test[feature])
            residual_test = self.X_test[feature] - clf_feature_test.predict(self.sens_test if is_cuf==False else self.counter_sens_test)
            residuals_dict_test[f"{feature}R"] = residual_test

        df_R_test = pd.DataFrame(residuals_dict_test)
        return df_R_test

    def predict(self, X_test=None):
        """function to predict using model"""
        if X_test is None:
            self.predictions = self.model.predict(self.X_test)
        else:
            self.predictions = self.model.predict(X_test)
        return self.predictions

    def predict_proba(self, X_test=None):
        """function to predict probability using model"""
        if X_test is None:
            self.pred_probs = self.model.predict_proba(self.X_test)[:,0]
        else:
            self.pred_probs = self.model.predict_proba(X_test)[:,0]
        return self.pred_probs

    def get_model_metrics(self):
        """function to get ROC AUC Score for model"""
        model_type = "ACF Model" if self.is_fair == True else "Full Model"
        sensitivity = " without sensitive features" if self.is_sensitive == False else " with sensitive features" if self.is_fair == False else ""
        metrics_data = pd.DataFrame({f"ROC AUC Score ({model_type}{sensitivity})":[roc_auc_score(self.y_test, self.pred_probs)]})
        return metrics_data

    def get_fairness_metrics(self):
        """function to get Equal Odds, Demographic Parity and Predictive Parity for model"""
        model_type = "ACF Model" if self.is_fair == True else "Full Model"
        sensitivity = " without sensitive features" if self.is_sensitive == False else " with sensitive features" if self.is_fair == False else ""
        fairness_metrics_dict = {"Column":[],f"Equal Odds ({model_type}{sensitivity})":[],f"Demographic Parity ({model_type}{sensitivity})":[],f"Predictive Parity ({model_type}{sensitivity})":[]}

        for feature in self.protected_features:
            fairness_metrics_dict["Column"].append(feature)
            tn_up, fp_up, fn_up, tp_up = confusion_matrix(self.y_test[self.X_test[feature]==1], self.predictions[self.X_test[feature]==1]).ravel()
            tn_p, fp_p, fn_p, tp_p = confusion_matrix(self.y_test[self.X_test[feature]==0], self.predictions[self.X_test[feature]==0]).ravel()
            fairness_metrics = acfmetrics(tn_up, fp_up, fn_up, tp_up, tn_p, fp_p, fn_p, tp_p)
            fairness_metrics_dict[f"Equal Odds ({model_type}{sensitivity})"].append(fairness_metrics[1])
            fairness_metrics_dict[f"Demographic Parity ({model_type}{sensitivity})"].append(fairness_metrics[3])
            fairness_metrics_dict[f"Predictive Parity ({model_type}{sensitivity})"].append(fairness_metrics[6])

        fairness_metrics_data = pd.DataFrame(fairness_metrics_dict)
        return fairness_metrics_data

    def get_model_metrics_difference(self):
        """function to get ROC AUC Score difference for protected features - privileged vs unprivileged"""
        model_type = "ACF Model" if self.is_fair == True else "Full Model"
        sensitivity = " without sensitive features" if self.is_sensitive == False else " with sensitive features" if self.is_fair == False else ""
        model_metrics_difference_dict = {"Column":[],f"ROC AUC Score Difference ({model_type}{sensitivity})":[]}

        for feature in self.protected_features:
            model_metrics_difference_dict["Column"].append(feature)
            A_fair=roc_auc_score(self.y_test[self.sens_test[feature]==0], self.pred_probs[self.sens_test[feature]==0]) #pval = 0 is Privileged
            B_fair=roc_auc_score(self.y_test[self.sens_test[feature]==1], self.pred_probs[self.sens_test[feature]==1]) #pval = 1 is Unprivileged
            model_metrics_difference_dict[f"ROC AUC Score Difference ({model_type}{sensitivity})"].append(abs(B_fair-A_fair))

        model_metrics_difference_data = pd.DataFrame(model_metrics_difference_dict)
        return model_metrics_difference_data

    def get_fairness_metrics_plots(self,metrics,acf_fairness_metrics,fm_fairness_metrics):
        """function to get fairness metrics comparison plots for protected features - Full Model VS ACF Model"""
        index = np.arange(len(metrics))
        bar_width = 0.35
        plt.figure(figsize=(50,25))

        for i,feature in enumerate(self.protected_features):
            ax = plt.subplot(2,2,i+1)
            fairness_metrics_dict = {"Metrics":metrics,'Full Model':fm_fairness_metrics.iloc[[i],[1,2,3]].values.tolist()[0], 'ACF Fair Model':acf_fairness_metrics.iloc[[i],[1,2,3]].values.tolist()[0]}
            fairness_metrics_table = pd.DataFrame.from_dict(fairness_metrics_dict)
            a = ax.bar(index, fairness_metrics_table["Full Model"], bar_width,color="red",label="Full Model")
            b = ax.bar(index+bar_width, fairness_metrics_table["ACF Fair Model"], bar_width, color="black",label="ACF Fair Model")
            ax.set_title(f"Fairness Metrics Comparison for Full Model vs ACF model for {feature}",fontsize=30)
            ax.set_xticks(index + bar_width / 2)
            ax.set_xticklabels(metrics,fontsize=30)
            ax.tick_params(axis='y', which='major', labelsize=30)
            ax.legend(loc='upper right',fontsize=30)
        plt.show()

    def get_crosstab_total_amount(self,predictions):
        crosstab = pd.crosstab(self.y_test, predictions, rownames=['Actual'], colnames=['Predicted']).rename(columns={0.0:"Non-Defaulter",1000.0:"Defaulter"}).rename({0.0:"Non-Defaulter",1000.0:"Defaulter"},axis="rows")

        df = pd.DataFrame()
        df["Default_Actual"] = pd.DataFrame(self.y_test.reset_index(drop=True))["Default"]
        df["Default_Predicted"] = pd.DataFrame(predictions)[0]
        df["AppliedAmount"] = pd.DataFrame(self.X_test["AppliedAmount"].reset_index(drop=True))

        TP = df[(df["Default_Actual"]==0) & (df["Default_Predicted"]==0)]["AppliedAmount"]
        FN = df[(df["Default_Actual"]==0) & (df["Default_Predicted"]==1)]["AppliedAmount"]
        FP = df[(df["Default_Actual"]==1) & (df["Default_Predicted"]==0)]["AppliedAmount"]
        TN = df[(df["Default_Actual"]==1) & (df["Default_Predicted"]==1)]["AppliedAmount"]

        total = sum(TP)-sum(FN)-sum(FP)
        return crosstab, total

    def get_error(self, predictions):
        """function to get error = diff(predicted,actual)"""
        error = self.y_test - predictions
        return error

    def get_counterfactual_data(self):
        """function to invert privileged and unprivileged classes in protected features"""
        self.counter_sens_test = self.sens_test.replace({0:1, 1:0})
        counter_X_test = pd.concat([self.X_test[self.independent_features], self.counter_sens_test], axis=1)
        return counter_X_test

    def get_plots_CUF(self, normal_predictions, cuf_predictions, normal_error, cuf_error):
        """function to get plots for model with and w/o CUF"""
        model_type = "ACF Fair Model" if self.is_fair == True else "Full Model"
        plt.figure(figsize=(8,5))
        p1=sns.kdeplot(normal_predictions, shade=True, color="r")
        p1=sns.kdeplot(cuf_predictions, shade=True, color="b")
        plt.title(f'Density plot of predictions for sensitive features VS counterfactual sensitive features for {model_type}', fontsize=10)
        plt.axvline(np.mean(normal_predictions), color="r")
        plt.axvline(np.mean(cuf_predictions), color="b")
        plt.legend(['Without CUF','With CUF'])