import os
import pandas as pd
from datetime import datetime
import os

os.chdir(os.path.dirname(os.path.abspath(__file__)))

ts = datetime.now().strftime("%Y%m%d%H%M%S")
mdPath = r'C:\git\2021-Legislation\analysis.md'

if os.path.isfile(mdPath):
    os.rename(mdPath, (mdPath + ts))

df = pd.read_csv (r'2021_LSR.csv')
df = df.sort_values(by=['BillNumber'])
for index, row in df.iterrows():
    if row.notna().BillAnalysis:
        analysis = str(row.BillAnalysis)
        bill = str(row.BillNumber)
        url = str(row.BillURL)
        output = "[" + bill + "](" + url + ") - " + analysis
        outfile = open(r'analysis.md',"a")
        outfile.write(output + "  \n")
        outfile.close()


from typing import List

from PyQt6.QtWidgets import QHBoxLayout

from services.player_store import OnAddPlayerSignalArgs, OnRemovePlayerSignalArgs
from services.service_locator import ServiceLocator
from widgets.player_widget import PlayerWidget

class PlayersContainer(QHBoxLayout):
    def __init__(self):
        super().__init__()

        self.setContentsMargins(0, 0, 0, 0)
        self.setSpacing(10)

        self._player_widgets: List[PlayerWidget] = []
        self._players_store = ServiceLocator.get().player
        self._players_store.on_add_player.connect(self._on_add_player_handler)
        self._players_store.on_remove_player.connect(self._on_remove_player_handler)

    def _on_add_player_handler(self, args: OnAddPlayerSignalArgs):
        player_widget = PlayerWidget(args.player)
        self._player_widgets.append(player_widget)
        self.addWidget(player_widget)

    def _on_remove_player_handler(self, args: OnRemovePlayerSignalArgs):
        index, player_widget = [
            (i, widget) for i, widget in enumerate(self._player_widgets) if widget.player == args.player
        ][0]
        self._player_widgets.remove(player_widget)
        self.itemAt(index).widget().deleteLater()


# Python imports
from unittest import TestCase, mock

# Third party imports
import pandas as pd

# Project imports
import constants.bq_utils as bq_consts
from tools import add_hpo
from tools.add_hpo import Path
from common import PIPELINE_TABLES, SITE_MASKING_TABLE_ID

class AddHPOTest(TestCase):

    @classmethod
    def setUpClass(cls):
        print('**************************************************************')
        print(cls.__name__)
        print('**************************************************************')

    def setUp(self):
        self.project_id = 'project_id'
        self.dataset_id = 'dataset_id'
        self.sandbox_dataset_id = 'sandbox_dataset_id'
        self.table_id = 'site_maskings'
        self.hpo_site_mappings_path = 'hpo_site_mappings_path'
        self.hpo_id_bucket_name_path = 'hpo_id_bucket_name_path'
        self.us_state = 'PIIState_XY'
        self.value_source_concept_id = 9999999
        self.hpo_id = 1010101

    def test_verify_hpo_site_info_up_to_date(self):
        df_1 = pd.DataFrame({'HPO_ID': ['FAKE_1', 'FAKE_2']})
        df_2 = pd.DataFrame({'hpo_id': ['fake_1', 'fake_2']})
        df_3 = pd.DataFrame({'hpo_id': ['fake_2', 'fake_3']})
        add_hpo.verify_hpo_site_info_up_to_date(df_1, df_2, 'site_mapping')
        df_4 = pd.DataFrame({'hpo_id': ['fake_1', 'fake_4']})
        self.assertRaises(ValueError, add_hpo.verify_hpo_site_info_up_to_date,
                          df_1, df_4, 'site_mapping')
        self.assertRaises(ValueError, add_hpo.verify_hpo_site_info_up_to_date,
                          df_2, df_4, 'bucket_name')
        self.assertRaises(ValueError, add_hpo.verify_hpo_site_info_up_to_date,
                          df_3, df_4, 'site_maskings')

    @mock.patch('bq_utils.get_hpo_info')
    @mock.patch('tools.add_hpo.pd.read_csv')
    def test_add_hpo_site_mappings_file_df(self, mock_read_csv, mock_hpo_info):
        mock_hpo_info.return_value = [{
            'hpo_id': 'fake_1',
            'name': 'fake_name_1'
        }, {
            'hpo_id': 'fake_2',
            'name': 'fake_name_2'
        }, {
            'hpo_id': 'fake_4',
            'name': 'fake_name_4'
        }]
        mock_read_csv.return_value = pd.DataFrame({
            'Org_ID': ['fake_org_1', 'fake_org_2', 'fake_org_4'],
            'HPO_ID': ['fake_1', 'fake_2', 'fake_4'],
            'Site_Name': ['fake_name_1', 'fake_name_2', 'fake_name_4'],
            'Display_Order': [1, 2, 3]
        })

        new_site = {
            'org_id': 'fake_org_3',
            'hpo_id': 'fake_3',
            'hpo_name': 'fake_name_3',
            'display_order': 3
        }
        actual_df = add_hpo.add_hpo_site_mappings_file_df(
            new_site['hpo_id'], new_site['hpo_name'], new_site['org_id'],
            self.hpo_site_mappings_path, new_site['display_order'])

        expected_df = pd.DataFrame({
            'Org_ID': ['fake_org_1', 'fake_org_2', 'fake_org_3', 'fake_org_4'],
            'HPO_ID': ['fake_1', 'fake_2', 'fake_3', 'fake_4'],
            'Site_Name': [
                'fake_name_1', 'fake_name_2', 'fake_name_3', 'fake_name_4'
            ],
            'Display_Order': [1, 2, 3, 4]
        })

        pd.testing.assert_frame_equal(actual_df.reset_index(drop=True),
                                      expected_df.reset_index(drop=True))

        self.assertRaises(ValueError, add_hpo.add_hpo_site_mappings_file_df,
                          new_site['hpo_id'], new_site['hpo_name'],
                          new_site['org_id'], self.hpo_site_mappings_path,
                          new_site['display_order'])

    @mock.patch('tools.add_hpo.BigQueryClient')
    def test_update_site_masking_table(self, mock_bq_client):
        # Mocks the job return
        query_job_reference_results = mock.MagicMock(
            name="query_job_reference_results")
        query_job_reference_results.return_value = query_job_reference_results
        query_job_reference_results.errors = []

        mock_call_response = mock.MagicMock(project=self.project_id)
        mock_call_response.query.return_value = query_job_reference_results
        mock_bq_client.return_value = mock_call_response
        mock_query = mock_bq_client.return_value.query

        mock_query.side_effect = query_job_reference_results

        # Test
        actual_job = add_hpo.update_site_masking_table(
            mock_bq_client(), self.hpo_id, self.us_state,
            self.value_source_concept_id)

        # Post conditions
        update_site_masking_query = add_hpo.UPDATE_SITE_MASKING_QUERY.render(
            project_id=self.project_id,
            pipeline_tables_dataset=PIPELINE_TABLES,
            site_maskings_table=SITE_MASKING_TABLE_ID,
            lookup_tables_dataset=bq_consts.LOOKUP_TABLES_DATASET_ID,
            hpo_site_id_mappings_table=bq_consts.HPO_SITE_ID_MAPPINGS_TABLE_ID,
            us_state=self.us_state,
            hpo_id=self.hpo_id,
            value_source_concept_id=self.value_source_concept_id)

        expected_job = query_job_reference_results
        mock_query.assert_any_call(update_site_masking_query)
        self.assertEqual(actual_job, expected_job)

    def test_check_state_code_format(self):
        us_state = 'XY'
        self.assertRaises(ValueError, add_hpo.check_state_code_format, us_state)

        us_state = 'PIIState_XY'
        add_hpo.check_state_code_format(us_state)

    def test_add_hpo_site_to_csv_files(self):
        new_site = {
            'org_id': 'fake_org',
            'hpo_id': 'fake_3',
            'hpo_name': 'fake_name',
            'display_order': 3,
            'bucket_name': 'fake_bucket',
            'us_state': 'PIIState_fake3',
            'value_source_concept_id': 1011010
        }
        hpo_site_csv_path = 'hpo_site_csv_files'
        mock_mapping_file_path = Path(
            hpo_site_csv_path) / bq_consts.MAPPING_CSV_FILE
        mock_bucket_file_path = Path(
            hpo_site_csv_path) / bq_consts.BUCKET_NAME_CSV_FILE
        mock_src_hpos_file_path = Path(
            hpo_site_csv_path) / bq_consts.SITE_MASKINGS_CSV_FILE

        with mock.patch.object(Path, 'is_file') as mock_is_file:
            mock_is_file.return_value = False
            mock_mapping_file_path.is_file()
            mock_bucket_file_path.is_file()
            mock_src_hpos_file_path.is_file()

            self.assertRaises(RuntimeError, add_hpo.add_hpo_site_to_csv_files,
                              new_site['hpo_id'], new_site['hpo_name'],
                              new_site['org_id'], new_site['bucket_name'],
                              hpo_site_csv_path, new_site['us_state'],
                              new_site['value_source_concept_id'],
                              new_site['display_order'])

            add_hpo.add_hpo_site_mappings_csv = mock.MagicMock()
            add_hpo.add_hpo_id_bucket_name_csv = mock.MagicMock()
            add_hpo.add_src_hpos_allowed_state_csv = mock.MagicMock()

            # Setting file exists at both mock_bucket_file_path and mock_mapping_file_path to True
            mock_is_file.return_value = True
            mock_mapping_file_path.is_file()
            mock_bucket_file_path.is_file()
            mock_src_hpos_file_path.is_file()

            add_hpo.add_hpo_site_to_csv_files(
                new_site['hpo_id'], new_site['hpo_name'], new_site['org_id'],
                new_site['bucket_name'], hpo_site_csv_path,
                new_site['us_state'], new_site['value_source_concept_id'],
                new_site['display_order'])

            add_hpo.add_hpo_site_mappings_csv.assert_called_with(
                new_site['hpo_id'], new_site['hpo_name'], new_site['org_id'],
                mock_mapping_file_path, new_site['display_order'])

            add_hpo.add_hpo_id_bucket_name_csv.assert_called_with(
                new_site['hpo_id'], new_site['bucket_name'],
                mock_bucket_file_path)

            add_hpo.add_src_hpos_allowed_state_csv.assert_called_with(
                new_site['hpo_id'], new_site['us_state'],
                new_site['value_source_concept_id'], mock_src_hpos_file_path)

    @mock.patch('bq_utils.get_hpo_bucket_info')
    @mock.patch('tools.add_hpo.pd.read_csv')
    def test_add_hpo_id_bucket_name_file_df(self, mock_read_csv,
                                            mock_hpo_bucket_info):
        mock_hpo_bucket_info.return_value = [{
            'hpo_id': 'fake_1',
            'bucket_name': 'fake_bucket_name_1'
        }, {
            'hpo_id': 'fake_2',
            'bucket_name': 'fake_bucket_name_2'
        }, {
            'hpo_id': 'fake_4',
            'bucket_name': 'fake_bucket_name_4'
        }]
        mock_read_csv.return_value = pd.DataFrame({
            'hpo_id': ['fake_1', 'fake_2', 'fake_4'],
            'bucket_name': [
                'fake_bucket_name_1', 'fake_bucket_name_2', 'fake_bucket_name_4'
            ],
            'service': ['default', 'default', 'default']
        })

        new_site = {
            'org_id': 'fake_org_3',
            'hpo_id': 'fake_3',
            'hpo_name': 'fake_name_3',
            'display_order': 3,
            'bucket_name': 'fake_bucket_name_3'
        }
        actual_df = add_hpo.add_hpo_id_bucket_name_file_df(
            new_site['hpo_id'], new_site['bucket_name'],
            self.hpo_id_bucket_name_path)

        expected_df = pd.DataFrame({
            'hpo_id': ['fake_1', 'fake_2', 'fake_4', 'fake_3'],
            'bucket_name': [
                'fake_bucket_name_1', 'fake_bucket_name_2',
                'fake_bucket_name_4', 'fake_bucket_name_3'
            ],
            'service': ['default', 'default', 'default', 'default']
        })

        pd.testing.assert_frame_equal(actual_df.reset_index(drop=True),
                                      expected_df.reset_index(drop=True))

        # Check for adding a site that exist in hpo_id_bucket_name.csv file.
        self.assertRaises(ValueError, add_hpo.add_hpo_id_bucket_name_file_df,
                          new_site['hpo_id'], new_site['bucket_name'],
                          self.hpo_id_bucket_name_path)

    @mock.patch('bq_utils.get_hpo_site_state_info')
    @mock.patch('tools.add_hpo.pd.read_csv')
    def test_add_src_hpos_allowed_state_file_df(self, mock_read_csv,
                                                mock_hpo_site_state_info):
        mock_hpo_site_state_info.return_value = [{
            'hpo_id': 'fake_1',
            'state': 'PIIState_fake1'
        }, {
            'hpo_id': 'fake_2',
            'state': 'PIIState_fake2'
        }, {
            'hpo_id': 'fake_4',
            'state': 'PIIState_fake4'
        }]
        mock_read_csv.return_value = pd.DataFrame({
            'State': ['PIIState_fake1', 'PIIState_fake2', 'PIIState_fake4'],
            'value_source_concept_id': [1101011, 1101010, 1011101],
            'hpo_id': ['fake_1', 'fake_2', 'fake_4']
        })

        new_site = {
            'org_id': 'fake_org_3',
            'hpo_id': 'fake_3',
            'hpo_name': 'fake_name_3',
            'display_order': 3,
            'bucket_name': 'fake_bucket_name_3',
            'us_state': 'PIIState_fake3',
            'value_source_concept_id': 1011010
        }
        actual_df = add_hpo.add_src_hpos_allowed_state_file_df(
            new_site['hpo_id'], new_site['us_state'],
            new_site['value_source_concept_id'], self.hpo_id_bucket_name_path)

        expected_df = pd.DataFrame({
            'State': [
                'PIIState_fake1', 'PIIState_fake2', 'PIIState_fake4',
                'PIIState_fake3'
            ],
            'value_source_concept_id': [1101011, 1101010, 1011101, 1011010],
            'hpo_id': ['fake_1', 'fake_2', 'fake_4', 'fake_3']
        })

        pd.testing.assert_frame_equal(actual_df.reset_index(drop=True),
                                      expected_df.reset_index(drop=True))

        # Check for adding a site that exist in src_hpos_to_allowed_states.csv file.
        self.assertRaises(ValueError,
                          add_hpo.add_src_hpos_allowed_state_file_df,
                          new_site['hpo_id'], new_site['us_state'],
                          new_site['value_source_concept_id'],
                          self.hpo_id_bucket_name_path)


from crispy_forms.helper import FormHelper
from crispy_forms.layout import Submit, Layout, Fieldset, ButtonHolder
from django import forms
from tournament_calculating.models import Group, ROUND_STATUS, ROUND_CONTUSIONS_STATUSES, ROUND_SPECIAL_STATUSES
from tournament_calculating.models import Participant, Fight, Round
from dal import autocomplete
from django.contrib.admin.widgets import AutocompleteSelectMultiple
from django.core.exceptions import ValidationError

class CreateParticipantForm(forms.ModelForm):

    class Meta:
        model = Participant
        fields = ['name', 'school', 'image']
        labels = {
            "name": "Imię i nazwisko",
            "school": "Szkoła",
            "image": "zdjęcie",
        }

        def __init__(self, *args, **kwargs):
            super().__init__(*args, **kwargs)
            self.helper = FormHelper()
            self.helper.form_method = 'post'
            self.helper.form_action = 'tournament_calculating: create_participant'
            self.helper.layout = Layout(
                Fieldset(
                    'Dodaj uczestnika',
                    'Imię i nazwisko',
                    'szkoła',
                    'zdjęcie'
                ),
                ButtonHolder(
                    Submit('submit', 'Dodaj', css_class='btn btn-primary'),
                    css_class="d-flex justify-content-end"
                )
            )

class AddParticipantForm(forms.ModelForm):

    class Meta:
        model = Group
        fields = ['participants']
        labels = {"participants": 'uczestnicy'}
        widgets = {
            'participants': autocomplete.ModelSelect2Multiple(url='tournament_calculating:participant-autocomplete')
        }

class AddRoundsForm(forms.ModelForm):

    class Meta:
        model = Fight
        fields = ['rounds']
        labels = {"rounds": 'starcia'}

        def __init__(self, *args, **kwargs):
            super().__init__(*args, **kwargs)
            self.helper = FormHelper()
            self.helper.form_method = 'post'
            self.helper.form_action = 'tournament_calculating:tournament_calculate'
            self.helper.layout = Layout(
                Fieldset(
                    'Dodaj ilość starć',
                    'rounds',
                ),
                ButtonHolder(
                    Submit('submit', 'Dodaj', css_class='btn btn-primary'),
                    css_class="d-flex justify-content-end"
                )
            )

class AddPointsForm(forms.ModelForm):

    class Meta:
        model = Round
        fields = ["points_fighter_one","points_fighter_two"]
        labels = {"points_fighter_one": 'punkty pierwszego zawodnika',
                  "points_fighter_two":'punkty drugiego zawodnika',
                  }

        # def clean(self):
        #     super(self).clean()
        #     points_fighter_one = self.cleaned_data.get(points_fighter_one)
        #     points_fighter_two = self.cleaned_data.get(points_fighter_two)
        #     if points_fighter_one in ['1','2','3','4','5'] and points_fighter_two in ['1','2','3','4','5']:
        #         self._errors['username'] = self.error_class([
        #             'Minimum 5 characters required'])

        def __init__(self, *args, **kwargs):
            super().__init__(*args, **kwargs)
            self.cleaned_data = None
            self.helper = FormHelper()
            self.helper.form_method = 'post'
            self.helper.form_action = 'tournament_calculating:add_points'
            self.helper.layout = Layout(
                Fieldset(
                    "punkty"
                ),
                ButtonHolder(
                    Submit('submit', 'Dodaj', css_class='btn btn-primary'),
                    css_class="d-flex justify-content-end"
                )
            )

    def clean_points_fighter_one(self, *args, **kwargs):

        points_fighter_one = self.cleaned_data.get("points_fighter_one")
        if points_fighter_one == 2:
            raise ValidationError("sprawdzam czy wyskoczył błąd, że jest 2")
        else:
            return points_fighter_one

class AddGroupForm(forms.ModelForm):

    class Meta:
        model = Group
        fields = ['color_fighter_one', 'color_fighter_two']
        labels = {"color_fighter_one": 'kolor opaski pierwszego zawodnika', "color_fighter_two": 'kolor opaski drugiego zawodnika'}

        def __init__(self, *args, **kwargs):
            super().__init__(*args, **kwargs)
            self.helper = FormHelper()
            self.helper.form_method = 'post'
            self.helper.form_action = 'tournament_calculating: add_group'
            self.helper.layout = Layout(
                Fieldset(
                    'Dodaj grupę',
                ),
                ButtonHolder(
                    Submit('submit', 'Dodaj', css_class='btn btn-primary'),
                    css_class="d-flex justify-content-end"
                )
            )

class AddFightsForm(forms.ModelForm):

    class Meta:
        model = Fight
        fields = ['rounds']
        labels = {"rounds": 'rundy'}

        def __init__(self, *args, **kwargs):
            super().__init__(*args, **kwargs)
            self.helper = FormHelper()
            self.helper.form_method = 'post'
            self.helper.form_action = 'tournament_calculating: add_fights'
            self.helper.layout = Layout(
                Fieldset(
                    'Dodaj walki',
                    'rundy',
                ),
                ButtonHolder(
                    Submit('submit', 'Dodaj', css_class='btn btn-primary'),
                    css_class="d-flex justify-content-end"
                )
            )

class GroupSummaryForm(forms.ModelForm):

    class Meta:
        model = Group
        fields = ['number_outgoing']
        labels = {'number_outgoing': 'ilość wychodzących z grupy'}

        def __init__(self, *args, **kwargs):
            super().__init__(*args, **kwargs)
            self.helper = FormHelper()
            self.helper.form_method = 'post'
            self.helper.form_action = 'tournament_calculating: group_summary'
            self.helper.layout = Layout(
                Fieldset(
                    'wychodzący z grupy'
                ),
                ButtonHolder(
                    Submit('submit', 'Dodaj', css_class='btn btn-primary'),
                    css_class="d-flex justify-content-end"
                )
            )


import numpy as np

def select_threshold(yval, pval):
    f1 = 0

    # You have to return these values correctly
    best_eps = 0
    best_f1 = 0

    for epsilon in np.linspace(np.min(pval), np.max(pval), num=1001):
        # ===================== Your Code Here =====================
        # Instructions: Compute the F1 score of choosing epsilon as the
        #               threshold and place the value in F1. The code at the
        #               end of the loop will compare the F1 score for this
        #               choice of epsilon and set it to be the best epsilon if
        #               it is better than the current choice of epsilon.
        #
        # Note : You can use predictions = pval < epsilon to get a binary vector
        #        of False(0)'s and True(1)'s of the outlier predictions
        #

        # ==========================================================

        if f1 > best_f1:
            best_f1 = f1
            best_eps = epsilon

    return best_eps, best_f1


import torch
from torch import nn
from torch.optim import Adam
import copy
from collections import deque
import random
import numpy as np
from torch.nn import functional as F

GAMMA = 0.999
TAU = 0.002
ACTOR_LR = 1e-3
CRITIC_LR = 1e-3
DEVICE = 'cuda'
BATCH_SIZE = 64

def soft_update(target, source):
    for tp, sp in zip(target.parameters(), source.parameters()):
        tp.data.copy_((1 - TAU) * tp.data + TAU * sp.data)

class Actor(nn.Module):
    def __init__(self, state_dim, action_dim):
        super().__init__()
        self.model = nn.Sequential(
            nn.Linear(state_dim, 256),
            nn.ELU(),
            nn.Linear(256, 256),
            nn.ELU(),
            nn.Linear(256, action_dim),
            nn.Tanh()
        )

    def forward(self, state):
        return self.model(state)

class Critic(nn.Module):
    def __init__(self, state_dim, action_dim):
        super().__init__()
        self.model = nn.Sequential(
            nn.Linear(state_dim + action_dim, 256),
            nn.ELU(),
            nn.Linear(256, 256),
            nn.ELU(),
            nn.Linear(256, 1)
        )

    def forward(self, state, action):
        return self.model(torch.cat([state, action], dim=-1)).view(-1)

class DDPG:
    def __init__(self, state_dim, action_dim):
        self.actor = Actor(state_dim, action_dim).to(DEVICE)
        self.critic = Critic(state_dim, action_dim).to(DEVICE)

        self.actor_optim = Adam(self.actor.parameters(), lr=ACTOR_LR)
        self.critic_optim = Adam(self.critic.parameters(), lr=CRITIC_LR)

        self.target_actor = copy.deepcopy(self.actor)
        self.target_critic = copy.deepcopy(self.critic)

        self.replay_buffer = deque(maxlen=100000)

        self.state_mean = None
        self.state_sq_mean = None

    def update(self, transition):
        self._append_transition(transition)
        if len(self.replay_buffer) > 16 * BATCH_SIZE:
            batch = [self.replay_buffer[random.randint(0, len(self.replay_buffer) - 1)] for _ in range(BATCH_SIZE)]
            state, action, next_state, reward = zip(*batch)
            state_std = self.state_sq_mean - self.state_mean ** 2
            state_std[state_std < 0] = 0
            state_std = state_std ** 0.5
            state = (np.array(state) - self.state_mean) / (state_std + 1e-16)
            state = torch.tensor(state, device=DEVICE, dtype=torch.float)
            action = torch.tensor(np.array(action), device=DEVICE, dtype=torch.float)
            next_state = (np.array(next_state) - self.state_mean) / (state_std + 1e-16)
            next_state = torch.tensor(next_state, device=DEVICE, dtype=torch.float)
            reward = torch.tensor(np.array(reward), device=DEVICE, dtype=torch.float)

            output = self.critic(state, action)
            with torch.no_grad():
                action_predicted = self.target_actor(next_state)
                target = reward + GAMMA * self.target_critic(next_state, action_predicted)
            loss = F.mse_loss(output, target)
            self.critic_optim.zero_grad()
            loss.backward()
            self.critic_optim.step()

            action_predicted = self.actor(state)
            loss = -torch.sum(self.critic(state, action_predicted))
            self.actor_optim.zero_grad()
            loss.backward()
            self.actor_optim.step()

            soft_update(self.target_actor, self.actor)
            soft_update(self.target_critic, self.critic)

    def _append_transition(self, transition):
        size = len(self.replay_buffer)
        state = transition[0]
        self.state_mean = state if size == 0 else (self.state_mean * size + state) / (size + 1)
        self.state_sq_mean = state ** 2 if size == 0 else (self.state_sq_mean * size + state ** 2) / (size + 1)
        self.replay_buffer.append(transition)

    def act(self, state):
        with torch.no_grad():
            if self.state_mean is not None:
                state_std = self.state_sq_mean - self.state_mean ** 2
                state_std[state_std < 0] = 0
                state_std = state_std ** 0.5
                state = (state - self.state_mean) / (state_std + 1e-16)
            state = torch.tensor(np.array([state]), device=DEVICE, dtype=torch.float)
            return self.actor(state).cpu().numpy()[0]

    def save(self):
        torch.save(self.actor, 'agent.pkl')


import torch
from data import data
import os
# import pandas as pd
# import numpy as np
from network_config import network_config

# 此文件用于计算empirical_kernel, 前提是已经训练了未近似的model.
# empirical_kernel是一个data_num x data_num的矩阵, 无法显示出来, 所以存到文件里

# 载入数据
# train_y = torch.tensor(data.train_data[:, 0].astype(int)).to(network_config.device) - 1
train_x = torch.tensor(data.train_data[:, 1:]).to(network_config.device)

# test_y = torch.tensor(data.test_data[:, 0].astype(int)).to(network_config.device) - 1
test_x = torch.tensor(data.test_data[:, 1:]).to(network_config.device)

# # 载入初始化参数
# init_net = torch.load(network_config.init_dir, map_location=network_config.device)

if not os.path.exists('./model/train_log/empirical_kernel'):
    os.mkdir('./model/train_log/empirical_kernel')
# 输出的是 empirical_kernel_m - empirical_kernel_0 的范数
write_result = open('./model/train_log/output_approx.csv', 'w')
write_result.write('epoch, theta_m - theta_0\n')

# specify i, j, 因为算整个empirical kernel太耗时间了, 所以我们选择特定的两个元素计算吧. 这里只给出一个例子i, j = 0, 1
i, j = 4, 1
for epoch in range(network_config.total_epoch):
    if epoch % network_config.record_interval == 0:
        # 当前epoch的模型参数保存路径
        original_net_dir = os.path.join(network_config.original_rnn_dir, 'epoch_{:0>5d}_original.pth'.format(epoch))
        # linear_nn_dir = os.path.join(network_config.linear_rnn_dir, 'epoch_{:0>5d}_linear.pth'.format(epoch))
        # 载入近似与非近似的网络的结构
        original_net = torch.load(original_net_dir, map_location=network_config.device)
        # linear_net = torch.load(linear_nn_dir, map_location=network_config.device)

        # 计算empirical kernel
        y = original_net(train_x)  # N x C
        # 因为pytorch只能对标量求导,所以要把上面的y矢量构造成一个标量
        temp = torch.ones(data.train_num, 1, dtype=torch.float, device=network_config.device, requires_grad=True)  # N x 1
        t = torch.mm(temp, torch.ones(1, data.y_dim, dtype=torch.float, device=network_config.device, requires_grad=False))  # 1 x C

        fc_hi_w_grad = torch.autograd.grad(y, original_net.fc_hi.weight, grad_outputs=t, create_graph=True)[0]
        fc_hi_b_grad = torch.autograd.grad(y, original_net.fc_hi.bias, grad_outputs=t, create_graph=True)[0]

        fc_hh_w_grad = torch.autograd.grad(y, original_net.fc_hh.weight, grad_outputs=t, create_graph=True)[0]

        fc_yh_w_grad = torch.autograd.grad(y, original_net.fc_yh.weight, grad_outputs=t, create_graph=True)[0]
        fc_yh_b_grad = torch.autograd.grad(y, original_net.fc_yh.bias, grad_outputs=t, create_graph=True)[0]

        h0_grad = torch.autograd.grad(y, original_net.h0, grad_outputs=t, create_graph=True)[0]

        g = torch.sum(torch.mul(fc_hi_w_grad, fc_hi_w_grad)) + torch.sum(torch.mul(fc_hi_b_grad, fc_hi_b_grad)) + torch.sum(torch.mul(fc_hh_w_grad, fc_hh_w_grad)) + torch.sum(torch.mul(fc_yh_w_grad, fc_yh_w_grad)) + torch.sum(torch.mul(fc_yh_b_grad, fc_yh_b_grad)) + torch.sum(torch.mul(h0_grad, h0_grad))

        dg = torch.autograd.grad(g, temp, create_graph=True)[0][j, 0]
        # print(dg)
        theta = torch.autograd.grad(dg, temp)[0][i, 0]

        print('epoch:{:05d}, theta:{:.10f}'.format(epoch, theta.item()))
        write_result.write('{:05d}, {:.10f}'.format(epoch, theta.item()))

        del original_net, y, temp, fc_hi_w_grad, fc_hi_b_grad, fc_hh_w_grad, fc_yh_w_grad, fc_yh_b_grad, h0_grad, g, dg, theta
        torch.cuda.empty_cache()

write_result.close()


import numpy as np
from sklearn import datasets
# import matplotlib.pyplot as plt

class SVC:
    def __init__(self, learning_rate: float = 0.001, lambda_param: float = 0.01, max_iter: int = 1000) -> None:
        self.lr = learning_rate
        self.lambda_param = lambda_param
        self.max_iter = max_iter
        self.weight = None
        self.bias = None

    def fit(self, X, y):
        y_ = np.where(y <= 0, -1, 1)
        n_samples, n_features = X.shape

        self.weight = np.zeros(n_features)
        self.bias = 0

        for _ in range(self.max_iter):
            for idx, xi in enumerate(X):
                gt_one = y_[idx] * (np.dot(xi, self.weight) - self.bias) >= 1
                if gt_one:
                    # update weight and bias
                    self.weight -= self.lr * (2 * self.lambda_param * self.weight)
                else:
                    self.weight -= self.lr * (2 * self.lambda_param * self.weight - np.dot(xi, y_[idx]))
                    self.bias -= self.lr * y_[idx]

    def predict(self, X):
        output = np.dot(X, self.weight) - self.bias
        return np.sign(output)  # The sign function returns -1 if x < 0, 0 if x==0, 1 if x > 0

if __name__ == '__main__':
    X, y = datasets.make_blobs(n_samples=50, n_features=2, centers=2, cluster_std=1.05)
    y = np.where(y == 0, -1, 1)

    svc_clf = SVC()
    svc_clf.fit(X, y)
    y_pred = svc_clf.predict(X)
    accuracy = np.mean(y == y_pred)

    print(f'Accuracy score: {accuracy * 100:.2f}%')
    print(svc_clf.weight, svc_clf.bias)

### Vizualise the predictions with hiperplanes ###

# def visualize_svm():
#     def get_hyperplane_value(x, w, b, offset):
#         return (-w[0] * x + b + offset) / w[1]
#
#     fig = plt.figure()
#     ax = fig.add_subplot(1, 1, 1)
#     plt.scatter(X[:, 0], X[:, 1], marker="o", c=y)
#
#     x0_1 = np.amin(X[:, 0])
#     x0_2 = np.amax(X[:, 0])
#
#     x1_1 = get_hyperplane_value(x0_1, clf.w, clf.b, 0)
#     x1_2 = get_hyperplane_value(x0_2, clf.w, clf.b, 0)
#
#     x1_1_m = get_hyperplane_value(x0_1, clf.w, clf.b, -1)
#     x1_2_m = get_hyperplane_value(x0_2, clf.w, clf.b, -1)
#
#     x1_1_p = get_hyperplane_value(x0_1, clf.w, clf.b, 1)
#     x1_2_p = get_hyperplane_value(x0_2, clf.w, clf.b, 1)
#
#     ax.plot([x0_1, x0_2], [x1_1, x1_2], "y--")
#     ax.plot([x0_1, x0_2], [x1_1_m, x1_2_m], "k")
#     ax.plot([x0_1, x0_2], [x1_1_p, x1_2_p], "k")
#
#     x1_min = np.amin(X[:, 1])
#     x1_max = np.amax(X[:, 1])
#     ax.set_ylim([x1_min - 3, x1_max + 3])
#
#     plt.show()
#
#
# visualize_svm()


#!/usr/bin/env python3
"""
module which containts dense_block function
"""
import tensorflow.keras as K
dense_block = __import__('5-dense_block').dense_block
transition_layer = __import__('6-transition_layer').transition_layer

def densenet121(growth_rate=32, compression=1.0):
    """
    builds the DenseNet-121 architecture as described in the paper
    "Densely Connected Convolutional Networks":

    * growth_rate is the growth rate
    * compression is the compression factor
    * You can assume the input data will have shape (224, 224, 3)
    * All convolutions should be preceded by Batch Normalization and
      a rectified linear activation (ReLU), respectively
    * All weights should use he normal initialization
    * Returns: the keras model
    """
    het_et_al = K.initializers.HeNormal()
    nb_filters = growth_rate * 2
    X = K.Input(shape=(224, 224, 3))

    x = K.layers.BatchNormalization()(X)
    x = K.layers.Activation('relu')(x)
    x = K.layers.Conv2D(nb_filters, kernel_size=(7, 7),
                        kernel_initializer=het_et_al,
                        strides=(2, 2), padding='same')(x)
    x = K.layers.MaxPool2D(pool_size=(3, 3), padding='same',
                           strides=(2, 2))(x)

    for i, layers in enumerate([6, 12, 24, 16]):
        x, nb_filters = dense_block(x, nb_filters, growth_rate, layers)
        if i != 3:
            x, nb_filters = transition_layer(x, nb_filters, compression)

    x = K.layers.AveragePooling2D(pool_size=(7, 7), padding='same')(x)
    x = K.layers.Dense(1000, activation='softmax',
                       kernel_initializer=het_et_al)(x)
    return K.Model(X, x)


import datetime
from flask import jsonify
import re
from app.models.db import DatabaseConnection

database=DatabaseConnection()
cur=database.cursor

class Message:

    """Messages Models contains properties stored for a message to be sent to an individual"""
    def __init__(self, subject, message, status, createdby, address, parentMessageId):
        """Initializes the message"""
        self.createdon=datetime.datetime.now()
        self.subject=subject
        self.message=message
        self.status=status
        self.createdby=createdby
        self.address=address
        self.parentMessageId=parentMessageId

    def create_message(self):
        query = "INSERT INTO messages(createdon, subject, message, status, createdby, address)\
            VALUES('{}', '{}', '{}', '{}', '{}', '{}')RETURNING *;".format(
                self.createdon,
                self.subject,
                self.message,
                self.status,
                self.createdby,
                self.address)
        cur.execute(query)
        return cur.fetchone()

    @staticmethod
    def find_message_by_id(id, email):
        query = "SELECT * FROM messages WHERE message_id = '{}' AND createdby='{}' OR address='{}'".format(id, email, email)
        cur.execute(query)
        return cur.fetchone()

    @staticmethod
    def delete_message(id, address):
        query = "DELETE FROM messages WHERE message_id = {} AND address ='{}'".format(id, address)
        cur.execute(query)

    @staticmethod
    def update_status(table_name, status, message_id):
        query = "UPDATE {} SET status = '{}' WHERE message_id = '{}';".format(table_name, status, message_id)
        cur.execute(query)

    @staticmethod
    def get_all_messages():
        query = "SELECT * FROM messages WHERE "
        cur.execute(query)
        return cur.fetchall()

    @staticmethod
    def get_unread_messages(email):
        """Get unread messages"""
        query="SELECT * FROM messages WHERE address='{}' AND status='sent';".format(email)
        cur.execute(query)
        return cur.fetchall()

    @staticmethod
    def get_all_messages_sent_by_a_user(email):
        """Get all user sent messages"""
        query="SELECT * FROM messages WHERE createdby='{}';".format(email)
        cur.execute(query)
        return cur.fetchall()

    @staticmethod
    def get_received_messages(address):
        query="SELECT * FROM messages WHERE address='{}';".format(address)
        cur.execute(query)
        return cur.fetchall()

    @staticmethod
    def get_id_from_email(address):
        query="SELECT message_id FROM messages WHERE address='{}'".format(address)
        cur.execute(query)
        return cur.fetchone()


#
# @lc app=leetcode id=69 lang=python3
#
# [69] Sqrt(x)
#

# @lc code=start
class Solution:
    def mySqrt(self, x: int) -> int:
        if x == 0:
            return 0

        left = 10 ** ((len(str(x)) - 1) // 2)
        right = left * 10

        while left <= right:
            mid = (left + right) // 2

            if mid ** 2 <= x < (mid + 1) ** 2:
                return mid
            elif mid ** 2 > x:
                right = mid - 1
            else:
                left = mid + 1

# @lc code=end

r = Solution().mySqrt(10)
print(r)


import threading, socket, logging, time, asyncio, json, datetime, random
from interfaces import System, Data, Event, CryptoHandler
from db import MySQL

class TcpServer(threading.Thread):

    _DEFAULT_TCP_SETTINGS = {"serverAddress" : socket.gethostname(), "serverPort" : 1234, "RSA" : 1024}
    _HANDSHAKE_REQUEST = b"199"
    _TCP_ACK_OK = b"200"
    _TCP_ACK_ERROR = b"400"
    _TCP_DEVICE_INFO_REQUEST = b"210"
    _TCP_SET_DEVICE_UID = b"220"
    _TCP_TERMINATOR = ["\r\r", b"\r\r"]

    def __init__(self, data: object, event: object, system: object, logger: object):
        if (isinstance(event, Event) != True  or isinstance(data, Data) != True
            or isinstance(system, System) != True or isinstance(logger, logging.Logger) == False):
            raise TypeError

        self._event = event
        self._data = data
        self._system = system
        self._logger = logger
        self._db = MySQL(system = system, logger = logger)

        for item in self._DEFAULT_TCP_SETTINGS.keys():                              # If this thread's settings don't exist, create them from default ones
            if item not in self._system.settings:
                self._system.updateSettings(newSettings = {item : self._DEFAULT_TCP_SETTINGS[item]})

        self._isRunning = True

        # Asyncio objects
        self._server = None
        self._asyncLoop = None
        self._asyncApp = None
        self._periodicDBTask = None

        (self._srvPubKey, self._srvPrivKey) = CryptoHandler.generateRSA(length = self._system.settings["RSA"])
        super().__init__(daemon = False, name = "TCP Server")

    def _openServer(self) -> bool:
        '''Open a server side endpoint. If this is not possible, kill this thread'''
        try:
            self._asyncLoop = asyncio.new_event_loop()
            asyncio.set_event_loop(self._asyncLoop)
            self._asyncApp = asyncio.start_server(self._handle_connection, self._system.settings["serverAddress"], self._system.settings["serverPort"], loop = self._asyncLoop)
            self._server = self._asyncLoop.run_until_complete(self._asyncApp)
            return True
        except Exception:
            self._logger.error("Impossible to open TCP server, error in async loop creation", exc_info = True)
            return False

    async def _handshake(self, reader: object, writer: object) -> tuple:
        '''Execute connection's handshake. Return the keys in case of success, otherwise return (None, None)'''

        try:
            writer.write(self._HANDSHAKE_REQUEST + self._TCP_TERMINATOR[1])                 # Send handshake request to the client
            await writer.drain()

            # Client has got the request and it's ready
            if await asyncio.wait_for(reader.readuntil(separator = self._TCP_TERMINATOR[1]), timeout = 5) == self._TCP_ACK_OK + self._TCP_TERMINATOR[1]:

                writer.write(str(self._system.settings["RSA"]).encode() + self._TCP_TERMINATOR[1])            # Send the RSA key length. This connection will use this key length
                await writer.drain()

                # Key length received and adopted
                if await asyncio.wait_for(reader.readuntil(separator = self._TCP_TERMINATOR[1]), timeout = 5) == self._TCP_ACK_OK + self._TCP_TERMINATOR[1]:
                    writer.write(CryptoHandler.exportRSApub(pubkey = self._srvPubKey) + self._TCP_TERMINATOR[1])        # Send the server's RSA public key
                    await writer.drain()

                    # Receive the AES key and use the server's private RSA key to decrypt is
                    aesKey = await asyncio.wait_for(reader.readuntil(separator = self._TCP_TERMINATOR[1]), timeout = 5)
                    aesKey = CryptoHandler.RSAdecrypt(privkey = self._srvPrivKey, secret = aesKey[0:-2], skipDecoding = True)

                    if aesKey != None:                                                      # AES key received and successfully decrypted
                        writer.write(self._TCP_ACK_OK + self._TCP_TERMINATOR[1])            # Send ACK
                        await writer.drain()

                        # Receive the client RSA public key and use the AES key to decrypt it
                        cltPubKey = await asyncio.wait_for(reader.readuntil(separator = self._TCP_TERMINATOR[1]), timeout = 5)
                        cltPubKey = CryptoHandler.AESdecrypt(key = aesKey, secret = cltPubKey[0:-2], byteObject = True)

                        if cltPubKey != False:                                              # If the RSA public key is successfully received
                            cltPubKey = CryptoHandler.importRSApub(PEMfile = cltPubKey)     # Import it

                            writer.write(self._TCP_ACK_OK + self._TCP_TERMINATOR[1])        # Handshake completed successfully, send the last ACK
                            await writer.drain()

                            return (cltPubKey, aesKey)
                        else:
                            self._logger.debug("Handshake failed in step 4 for client " + str(writer.get_extra_info("peername")))
                    else:
                        self._logger.debug("Handshake failed in step 3 for client " + str(writer.get_extra_info("peername")))
                else:
                    self._logger.debug("Handshake failed in step 2 for client " + str(writer.get_extra_info("peername")))
            else:
                self._logger.debug("Handshake failed in step 1 for client " + str(writer.get_extra_info("peername")))

            # Something went wrong, handshake failed
            self._logger.warning("Handshake failed for client " + str(writer.get_extra_info("peername")))
            return (None, None)
        except asyncio.TimeoutError:                                                        # At some point, client didn't send anythingù
            self._logger.warning("Timeout connection in handshake for client " + str(writer.get_extra_info("peername")))
            return (None, None)
        except Exception:                                                                   # An error occurred, handshake failed
            self._logger.error("Error occurred during TCP handshake", exc_info = True)
            return (None, None)

    async def _send(self, data: object, aesKey: object, writer: object, byteObject: bool = False) -> None:
        if type(byteObject) != bool:
            raise TypeError

        data = CryptoHandler.AESencrypt(key = aesKey, raw = data, byteObject = byteObject) + self._TCP_TERMINATOR[1]
        writer.write(data)
        await writer.drain()
        return

    async def _recv(self, aesKey: object, reader: object, byteObject: bool = False) -> object:
        if type(byteObject) != bool:
            raise TypeError

        data = await asyncio.wait_for(reader.readuntil(separator = self._TCP_TERMINATOR[1]), timeout = 5)
        data = CryptoHandler.AESdecrypt(key = aesKey, secret = data[0:-2], byteObject = byteObject)
        return data

    async def _handle_connection(self, reader, writer) -> None:
        '''Handle a TCP client connection'''

        time.sleep(0.1)                                                                     # Small delay to let the client sync
        (cltPubKey, aesKey) = await self._handshake(reader, writer)

        if cltPubKey != None and aesKey != None:
            try:
                clientUID = await self._recv(aesKey = aesKey, reader = reader, byteObject = False)
                if len(clientUID) == 10:
                    await self._send(data = self._TCP_ACK_OK, aesKey = aesKey, writer = writer, byteObject = True)
                    sensorData = await self._recv(aesKey = aesKey, reader = reader, byteObject = False)
                    sensorData = json.loads(sensorData)
                    if type(sensorData) == dict:
                        # Now that we have both UID and data, let's check if there is something to do
                        if int(clientUID) == 0 or len(str(clientUID)) != 10:                                             # This device has never been configured
                            try:
                                # Request info about this device
                                await self._send(data = self._TCP_DEVICE_INFO_REQUEST, aesKey = aesKey, writer = writer, byteObject = True)

                                # Decode the info
                                try:
                                    deviceInfo = await self._recv(aesKey = aesKey, reader = reader, byteObject = False)
                                    deviceInfo = json.loads(deviceInfo)
                                except Exception:
                                    self._logger.debug("Device info not received")
                                    raise Exception

                                # Check in the DB
                                try:
                                    storedDeviceInfo = await self._db.getSensorInfo(city = deviceInfo["City"], country = deviceInfo["Country"])
                                except KeyError:
                                    self._logger.debug("Received device info are in a broken format")
                                    storedDeviceInfo = None
                                except Exception:
                                    self._logger.warning("An error occurred while getting device info")
                                    storedDeviceInfo = None

                                if storedDeviceInfo == tuple():                             # This device do not exist in our system
                                    newUID = str(random.randint(1000000000, 9999999999))    # Generate a new random UID for this device
                                    try:                                                    # Record this device into our system
                                        await self._insertDataIntoDB(tableName = "Devices", data = {newUID : {"country" : deviceInfo["Country"], "city" : deviceInfo["City"]}})
                                    except Exception:                                       # We couldn't record this device so we can't go on
                                        self._logger.warning("An error occurred while recording a new device into the DB")
                                        await self._send(data = self._TCP_ACK_OK, aesKey = aesKey, writer = writer, byteObject = True)
                                        writer.close()
                                        await writer.wait_closed()
                                        return
                                    clientUID = newUID                                      # Data received from this device must be recorded with the new UID
                                elif storedDeviceInfo == None:                              # We couldn't get info from this device so we can't record these data, just kill the connection
                                    await self._send(data = self._TCP_ACK_OK, aesKey = aesKey, writer = writer, byteObject = True)
                                    writer.close()
                                    await writer.wait_closed()
                                    self._logger.debug("Connection closed with client " + str(writer.get_extra_info("peername")))
                                    return
                                else:
                                    clientUID = str(storedDeviceInfo[0])

                                # Now that we have a valid UID to set, notify our intention
                                await self._send(data = self._TCP_SET_DEVICE_UID, aesKey = aesKey, writer = writer, byteObject = True)

                                # Wait an ACK from the device. If it's arrive, send the new UID
                                try:
                                    answer = await self._recv(aesKey = aesKey, reader = reader, byteObject = True)
                                except Exception:
                                    answer = None

                                if answer == self._TCP_ACK_OK:
                                    await self._send(data = clientUID, aesKey = aesKey, writer = writer, byteObject = False)

                                    # Check if everything is okay
                                    try:
                                        answer = await self._recv(aesKey = aesKey, reader = reader, byteObject = True)
                                    except Exception:
                                        answer = None

                                    if answer != self._TCP_ACK_OK:
                                        self._logger.debug("Last ACK for update UID not received")
                                        raise Exception
                                else:
                                    self._logger.debug("First ACK for update UID not received")
                                    raise Exception
                            except Exception:
                                self._logger.warning("Impossible to configure a device. Data lost")
                                await self._send(data = self._TCP_ACK_OK, aesKey = aesKey, writer = writer, byteObject = True)
                                writer.close()
                                await writer.wait_closed()
                                self._logger.debug("Connection closed with client " + str(writer.get_extra_info("peername")))
                                return
                        elif int(clientUID) != 0 and len(str(clientUID)) == 10:             # This client is configured, let's check if it exists in our system
                                # Check in the DB
                                try:
                                    storedDeviceInfo = await self._db.getSensorInfo(UID = int(clientUID))
                                except KeyError:
                                    self._logger.debug("Received device info are in a broken format")
                                    storedDeviceInfo = None
                                except Exception:
                                    self._logger.warning("An error occurred while getting device info")
                                    storedDeviceInfo = None

                                # If we don't have this device in our DB, request info and store it
                                if storedDeviceInfo == None:
                                    # Request info about this device
                                    await self._send(data = self._TCP_DEVICE_INFO_REQUEST, aesKey = aesKey, writer = writer, byteObject = True)

                                    # Decode the info
                                    try:
                                        deviceInfo = await self._recv(aesKey = aesKey, reader = reader, byteObject = False)
                                        deviceInfo = json.loads(deviceInfo)
                                    except Exception:
                                        self._logger.debug("Device info not received")
                                        raise Exception

                                    try:                                                    # Record this device into our system
                                        await self._insertDataIntoDB(tableName = "Devices", data = {clientUID : {"country" : deviceInfo["Country"], "city" : deviceInfo["City"]}})
                                    except Exception:                                       # We couldn't record this device so we can't go on
                                        self._logger.warning("An error occurred while recording a new device into the DB")
                                        await self._send(data = self._TCP_ACK_OK, aesKey = aesKey, writer = writer, byteObject = True)
                                        writer.close()
                                        await writer.wait_closed()
                                        return

                        # Create a new task in background to insert data into the DB without blocking
                        self._asyncLoop.create_task(self._insertDataIntoDB(tableName = "Recordings", data = {"UID" : clientUID, "values" : sensorData}))
                        await self._send(data = self._TCP_ACK_OK, aesKey = aesKey, writer = writer, byteObject = True)
                        self._logger.debug("Data recorded for device UID = " + str(clientUID))
                    else:
                        await self._send(data = self._TCP_ACK_ERROR, aesKey = aesKey, writer = writer, byteObject = True)
                        self._logger.warning("Sensors data are unreadable from client " + str(writer.get_extra_info("peername")))

                    writer.close()
                    await writer.wait_closed()
                    self._logger.debug("Connection closed with client " + str(writer.get_extra_info("peername")))
                    return
            except asyncio.TimeoutError:                        # At some point, the client didn't responde, close the connection
                self._logger.warning("Timeout connection for client " + str(writer.get_extra_info("peername")))
                writer.close()
                await writer.wait_closed()
                return
            except Exception:                                   # Unknown error occurred, close the connection
                self._logger.error("Error occurred while talking with client " + str(writer.get_extra_info("peername")), exc_info = True)
                writer.close()
                await writer.wait_closed()
                return
        else:
            self._logger.debug("TCP handshake failed for client " + str(writer.get_extra_info("peername")))
            writer.close()
            await writer.wait_closed()
            return

    async def _insertDataIntoDB(self, tableName: str, data: dict) -> None:
        '''
        Insert the given data into the specified table. In case of error, try to rebuild the DB connection
        '''

        if type(tableName) != str or type(data) != dict:
            raise TypeError

        try:
            if await self._db.insertData(tableName = tableName, data = data) == False:          # Something went wrong with the connection
                self._logger.warning("An error occurred while inserting data into DB in table {table}".format(table = tableName))
                self._db.close()
                self._db.open()
                await self._db.insertData(tableName = tableName, data = data)                   # Try to insert these data again
        except Exception:
            self._logger.error("Unexpected error while inserting data into DB", exc_info = True)

    async def _optimizeDB(self) -> None:
        '''
        This coroutine execute once every 24 hours to optimize the DB usage by reducing the records resolution
        '''

        try:
            while self._isRunning == True:
                currentTime = datetime.datetime.now()                                       # Get the current time in order to understand which optimization has to be done

                if currentTime.month == 1 and currentTime.day == 1:                         # Today is 01/01
                    if await self._db.optimizeDB(tableName = "Recordings", option = {"period" : "year"}) == True:
                        self._logger.debug("Year wide optimization has been successfull")
                    else:
                        self._logger.warning("Year wide optimization failed")

                if currentTime.day == 1:                                                    # First day of the month
                    if await self._db.optimizeDB(tableName = "Recordings", option = {"period" : "month"}) == True:
                        self._logger.debug("Month wide optimization has been successfull")
                    else:
                        self._logger.warning("Month wide optimization failed")

                if await self._db.optimizeDB(tableName = "Recordings", option = {"period" : "day"}) == True:
                    self._logger.debug("Day wide optimization has been successfull")
                else:
                    self._logger.warning("Day wide optimization failed")

                await asyncio.sleep(delay = 86400)                                  # Sleep for 24 hours before repeating this procedure
        except asyncio.CancelledError:
            self._logger.debug("Periodic DB optimization routine terminated gracefully")
        except Exception:
            self._logger.critical("Periodic DB optimization routine terminated with errors", exc_info = True)
        return

    def stopThread(self) -> None:
        '''Close the server endpoint'''
        self._isRunning = False                                                     # Update the status flag

        if self._asyncLoop != None:                                                 # If the async loop already exists
            if self._periodicDBTask != None:                                        # There is the periodic DB optimization routine
                self._asyncLoop.call_soon_threadsafe(self._periodicDBTask.cancel)

            self._asyncLoop.stop()                                                  # Stop the async loop. This won't wake up the system so let's fake a connection

            # The async loop is stopped so this connection will wake up the async ThreadPoolExecutor which will then terminate
            try:
                fakeClient = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                fakeClient.connect((self._system.settings["serverAddress"], self._system.settings["serverPort"]))
                fakeClient.close()
            except OSError:                                                         # Impossible to establish a connection, it means that the server is already closed
                pass
            except Exception:                                                       # Unknown exception
                self._logger.error("Something happened while trying to close the TCP server throught a fake connection")

            self._db.close()                                                        # Close the connection to DB
        return

    def run(self) -> None:
        while True:
            if self._isRunning == False:                                            # Check the thread status
                try:
                    self._server.close()                                            # Close the server coroutine
                    self._asyncLoop.run_until_complete(self._server.wait_closed())  # Wait for its termination
                except Exception:                                                   # Something went wrong
                    self._logger.critical("Error occurred while closing the TCP server async loop", exc_info = True)

                self._logger.debug("TCP Server closed")                             # Thread is dead, leave this function
                return
            else:                                                                   # Thread is alive
                if self._db.status == False:                                        # Check the DB handler status
                    if self._db.open() == False:                                    # If it was closed, try to open it
                        self._logger.critical("Fatal error: impossible to open DB") # In case of failure, the whole program is broken
                        self.stopThread()
                        continue

                if asyncio.run(self._db.createTable(tableName = "Recordings")) == False:         # Prepare the tables. In case of failure we can't go on
                    self._logger.critical("Fatal error: impossible to create the Recordings table into DB")
                    self.stopThread()
                    continue

                if self._openServer() == True:                                      # Try to open the async server
                    self._periodicDBTask = self._asyncLoop.create_task(self._optimizeDB())
                    self._asyncLoop.run_forever()                                   # In case of success, wait here the closure
                else:                                                               # Impossible to open the server
                    time.sleep(120)                                                 # Wait 2 minutes and then try again
                    continue


import sys
import csv
import math
import mpcc_or
import mpcc_chisq
import mpcc_fishertests
import shared_functions
rowcount = 0
oldpairid = "-1"

excontrol=0
excase=0

exex=0
exun=0
unex=0
unun=0

with open(sys.argv[1], newline='') as csvfile:
  sickreader = csv.reader(csvfile, delimiter=',', quotechar='"')
  next(sickreader)
  sortsick=sorted(sickreader, key=lambda x: (int(x[0]), int(x[1])), reverse=False)
  row_count = sum(1 for row in sortsick)
  for row in sortsick:
    pairid = row[0]
    print(' '.join(row))
    rowcount = rowcount + 1
    if pairid != oldpairid:
      if rowcount > 1:
        if excase == 1:
          if excontrol == 1:
            exex = exex + 1
        if excase == 1:
          if excontrol == 0:
            exun = exun + 1
        if excase == 0:
          if excontrol == 1:
            unex = unex + 1
        if excase == 0:
          if excontrol == 0:
            unun = unun + 1
        excontrol=0
        excase=0
    oldpairid = pairid
    if int(row[1]) == 1:
      if int(row[2]) == 1:
        excase = excase + 1
    if int(row[1]) == 0:
      if int(row[2]) == 1:
        excontrol = excontrol + 1
    if rowcount == row_count:
      if excase == 1:
        if excontrol == 1:
          exex = exex + 1
      if excase == 1:
        if excontrol == 0:
          exun = exun + 1
      if excase == 0:
        if excontrol == 1:
          unex = unex + 1
      if excase == 0:
        if excontrol == 0:
          unun = unun + 1

#rint('Both Exposed = ', exex)
#rint('Case Exposed, Control Unexposed = ', exun)
#rint('Case Unexposed, Control Exposed = ', unex)
#rint('Both Unexposed = ', unun)
print('==========================')
print(str(exex) + '\t\t' + str(exun))
print(str(unex) + '\t\t' + str(unun))
print('Odds Ratio = ' + str(mpcc_or.oddsratio(exun, unex)) + ' (' + "{:.4f}".format(mpcc_or.lcl(exun, unex)) + ', ' + "{:.4f}".format(mpcc_or.ucl(exun, unex)) + ')')
fisherlimits = [-9.9, -9.9]
mpcc_or.fisherlimits(exex, exun, unex, unun, fisherlimits)
print('\t\t(' + "{:.4f}".format(fisherlimits[0]) + ', ' + "{:.4f}".format(fisherlimits[1]) + ')')
mcnemar = mpcc_chisq.uncorrected(exun, unex)
print('McNemar = ' + "{:.4f}".format(mcnemar) + '; p = ' + "{:.4f}".format(shared_functions.pvalfromchisq(mcnemar, 1.0)))
mcncorrected = mpcc_chisq.corrected(exun, unex)
print('Corrected = ' + "{:.4f}".format(mcncorrected) + '; p = ' + "{:.4f}".format(shared_functions.pvalfromchisq(mcncorrected, 1.0)))
fishertests = [-9.9, -9.9]
mpcc_fishertests.fishertests(exex, exun, unex, unun, fishertests)
print('\t' + "{:.6f}".format(fishertests[0]) + '\t' + "{:.6f}".format(fishertests[1]))


from functions import dictfetchall, listfetchall, dictfetchone
from django.db import connection
from rest_framework import status
import functionssql as fs

def get_user_feed(user_id):
    with connection.cursor() as cursor:
        cursor.execute('SELECT * FROM article WHERE channel_id IN(SELECT channel_id FROM user_follows_channel WHERE user_id = %s) ORDER BY created_at DESC', [user_id])
        data = dictfetchall(cursor)
        for article in data:
            article['user'] = get_article_poster(article['article_id'])
            article['channel'] = get_article_channel(article['article_id'])
        return data

def get_nologin_feed():
    with connection.cursor() as cursor:
        cursor.execute('SELECT * FROM article ORDER BY created_at DESC')
        data = dictfetchall(cursor)
        for article in data:
            article['user'] = get_article_poster(article['article_id'])
            article['channel'] = get_article_channel(article['article_id'])
        return data

#Get article information
def get_article(article_id, cursor = connection.cursor()):
    cursor.execute("SELECT * FROM article WHERE article_id = %s", [article_id])
    data = dictfetchall(cursor)[0]
    return data

#Gets the top 10 liked articles in the month
def get_top_monthly_articles():
    with connection.cursor() as cursor:
        cursor.execute("""
            SELECT * FROM article WHERE article_id IN (
                SELECT article_id FROM user_likes_article
                WHERE TIMESTAMPDIFF(DAY, created_at, NOW()) < 31
                GROUP BY article_id
                ORDER BY count(*) DESC);
        """)
        articles = dictfetchall(cursor)
        articles = fs.articlelist_get_channel(articles)
        articles = fs.articlelist_get_user(articles)
        return articles

#Get article's poster information
def get_article_poster(article_id):
    try:
        with connection.cursor() as cursor:
            cursor.execute("SELECT user.* FROM user JOIN channel USING(user_id) JOIN article USING(channel_id) WHERE article_id = %s", [article_id])
            data = dictfetchall(cursor)[0]
    except Exception:
        print('sql.get_article_poster - cant find article\'s user')
        data = None
    return data

#Get article's poster and channel information
def get_article_channel(article_id):
    try:
        with connection.cursor() as cursor:
            cursor.execute("SELECT channel.* FROM channel JOIN article USING(channel_id) WHERE article_id = %s", [article_id])
            data = dictfetchall(cursor)[0]
    except Exception:
        print('sql.get_article_channel - cant find article\'s channel')
        data = None
    return data

#Get comments (and the user who posted them) on article
def get_article_comments(article_id):
    with connection.cursor() as cursor:
        cursor.execute("SELECT * FROM user LEFT JOIN comment USING(user_id) WHERE article_id = %s", [article_id])
        data = dictfetchall(cursor)
    return data

#Get if current user liked article
def get_user_liked_article(user_id, article_id):
    with connection.cursor() as cursor:
        cursor.execute("SELECT exists(SELECT 1 FROM user_likes_article WHERE user_id = %s AND article_id = %s) liked",
            [user_id, article_id])
        data = dictfetchall(cursor)[0]
        res = True if data['liked'] is 1 else False;
    return res

#Get number of likes on article
def get_article_likes(article_id):
    with connection.cursor() as cursor:
        cursor.execute("SELECT count(distinct user_id) likes FROM user_likes_article WHERE article_id = %s", [article_id])
        data = dictfetchall(cursor)[0]['likes']
    return data

#Update views on article
def create_view(article_id,user_id = None):
    with connection.cursor() as cursor:
        cursor.execute("INSERT INTO view(user_id,article_id) VALUES (%s, %s)", [user_id, article_id])
    return {}

#Creates an article based on given params (Will default to null to nullable fields)
def create_article(channel_id,url,caption = None,preview_image = None,preview_title = None,preview_text = None, num_words=None, shared_from_article_id = None, preview_x_frame_options = None):
    with connection.cursor() as cursor:
        cursor.execute('INSERT INTO article(channel_id,url,caption,preview_image,preview_title,preview_text,num_words, shared_from_article_id, preview_x_frame_options) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)', [channel_id,url,caption,preview_image,preview_title,preview_text,num_words,shared_from_article_id,preview_x_frame_options])
        cursor.execute('SELECT last_insert_id()')
        article_id = dictfetchone(cursor)['last_insert_id()']
        cursor.execute('SELECT user_id FROM user_follows_channel WHERE channel_id = %s', channel_id)
        channel_followers = listfetchall(cursor)
        cursor.execute('SELECT user_id from channel where channel_id = %s', channel_id)
        user_id = dictfetchone(cursor)['user_id']
        for follower in channel_followers:
            if follower != user_id:
                cursor.execute("""
                    INSERT INTO notification (type, article_id, channel_id, type_id, type_user_id, is_read, user_id)
                    VALUES ('article', %s, %s, %s, %s, false, %s)""", [article_id, channel_id, channel_id, user_id, follower])

    return {}

#Deletes an article given the article_id
def delete_article(article_id):
    with connection.cursor() as cursor:
        cursor.execute('DELETE FROM article WHERE article_id = %s' , [article_id])
    return {}

def create_like(article_id,user_id):
    with connection.cursor() as cursor:
        cursor.execute('INSERT INTO user_likes_article(article_id,user_id) VALUES (%s,%s)', [article_id,user_id])
    return {}

def delete_like(article_id,user_id):
    with connection.cursor() as cursor:
        cursor.execute('DELETE FROM user_likes_article WHERE article_id = %s AND user_id = %s', [article_id,user_id])
    return {}

#This to be used by create comment to immediately get created comment
#Look into last_inserted_id() why it doesnt work?
def create_get_comment(article_id, user_id):
    with connection.cursor() as cursor:
        cursor.execute('SELECT * FROM comment WHERE article_id = %s AND user_id = %s AND created_at=(SELECT max(created_at) FROM comment)', [article_id,user_id])
        data = dictfetchall(cursor)
    return data

def create_comment(article_id,user_id,comment_text):
    with connection.cursor() as cursor:
        cursor.execute('INSERT INTO comment(article_id,user_id,comment_text) VALUES (%s,%s,%s)', [article_id,user_id,comment_text])
        data = create_get_comment(article_id,user_id)
    return data

def get_comment(comment_id):
    try:
        with connection.cursor() as cursor:
            cursor.execute('SELECT * FROM comment WHERE comment_id = %s', [comment_id])
            data = dictfetchall(cursor)
    except Exception as e:
        return {'errorType':str(type(e)), 'errorArgs':e.args}, status.HTTP_500_INTERNAL_SERVER_ERROR
    return data, status.HTTP_200_OK


import copy
import json
import math
from typing import final
import urllib.request

from utils import *
from datetime import datetime
from collections import defaultdict

class Library:
    def __init__(self, rel_count, isDev=True):
        '''
        - Initialization function of the library instance.
        It sets many dictionaries to be used in many purposes.
        P.S.: `ix` used as index and for map dictionaries,
        first indicator is always the key, and second is value.
        e.g. self.book_ix_url_map is a dict with indexes as keys,
        and urls as values.
        '''
        self.env_dev = isDev
        self.prod_dir = "./books"
        self.test_dir = "./test_books"
        self.relevant_count = rel_count
        self.total_number_of_books = 0
        self.last_state = None
        self.alpha = 0.5
        self.book_is_added = defaultdict(bool)
        self.book_ix_url_map = defaultdict(str)
        self.book_url_ix_map = defaultdict(int)
        self.book_ix_genre_map = defaultdict(str)
        self.book_genre_ix_map = defaultdict(int)
        self.desc_dict = defaultdict(int)
        self.desc_corpus = defaultdict(dict)
        self.desc_tf_idf = defaultdict(dict)
        self.genre_dict = defaultdict(int)
        self.genre_corpus = defaultdict(dict)
        self.genre_tf_idf = defaultdict(dict)
        self.last_query_scores = defaultdict(dict)

    def load_booklist(self, path):
        '''
        - Loads the given booklist by the user into the system and
        calculates their tf-idf values.
        '''
        url_list = read_book_urls(path)
        for url in url_list:
            downloaded_book = self.download_a_book(url)
            self.add_to_library(downloaded_book)
        print("\r\n--> Books in the given list has been added.")
        self.calculate_tf_idf()
        print("--> Calculations have been completed.")
        self.dump_resources()
        print("--> All resources have been dumped.")

    def dump_resources(self):
        '''
        - Exports the current state of the library to a file to be
        imported whenever necessary.
        '''
        fout = open("resources.json", "w")

        current_datetime = datetime.utcnow()

        resources = {}
        resources["rel_count"] = copy.deepcopy(self.relevant_count)
        resources["total_books"] = copy.deepcopy(self.total_number_of_books)
        resources["book_is_added"] = copy.deepcopy(self.book_is_added)
        resources["book_ix_url_map"] = copy.deepcopy(self.book_ix_url_map)
        resources["book_url_ix_map"] = copy.deepcopy(self.book_url_ix_map)
        resources["book_ix_genre_map"] = copy.deepcopy(self.book_ix_genre_map)
        resources["book_genre_ix_map"] = copy.deepcopy(self.book_genre_ix_map)
        resources["desc_dict"] = copy.deepcopy(self.desc_dict)
        resources["desc_corpus"] = copy.deepcopy(self.desc_corpus)
        resources["desc_tf_idf"] = copy.deepcopy(self.desc_tf_idf)
        resources["genre_dict"] = copy.deepcopy(self.genre_dict)
        resources["genre_corpus"] = copy.deepcopy(self.genre_corpus)
        resources["genre_tf_idf"] = copy.deepcopy(self.genre_tf_idf)
        resources["last_state"] = copy.deepcopy(str(current_datetime))
        resources["last_query"] = copy.deepcopy(self.last_query_scores)

        json.dump(resources, fout)
        fout.flush()
        fout.close()
        print(f"--> System screenshot is taken at {current_datetime}")
        del resources

    def load_resources(self):
        '''
        - Import a system state screenshot to the system whenever desired.
        '''
        fin = open("resources.json", "r")

        imported_resources = json.load(fin)
        fin.close()
        self.relevant_count = imported_resources["rel_count"]
        self.total_number_of_books = imported_resources["total_books"]
        self.book_is_added = imported_resources["book_is_added"]
        self.book_ix_url_map = imported_resources["book_ix_url_map"]
        self.book_url_ix_map = imported_resources["book_url_ix_map"]
        self.book_ix_genre_map = imported_resources["book_ix_genre_map"]
        self.book_genre_ix_map = imported_resources["book_genre_ix_map"]
        self.desc_dict = imported_resources["desc_dict"]
        self.desc_corpus = imported_resources["desc_corpus"]
        self.desc_tf_idf = imported_resources["desc_tf_idf"]
        self.genre_dict = imported_resources["genre_dict"]
        self.genre_corpus = imported_resources["genre_corpus"]
        self.genre_tf_idf = imported_resources["genre_tf_idf"]
        self.last_state = imported_resources["last_state"]
        self.last_query_scores = imported_resources["last_query"]

    def add_to_library(self, _book):
        '''
        - Adds a book to the library, which means many dictionaries
        e.g. corpus, dictionary and maps. If a book is not downloaded
        before, it also downloads it.
        '''
        if isinstance(_book, Book):
            book = _book
        else:
            if _book in set(self.book_url_ix_map.keys()):
                print("--> This book is already added.")
                return
            else:
                book = self.download_a_book(_book)

        desc = book.book_description
        genre = book.book_genre_info
        tokens = processed_string(desc)
        unique_words = set(tokens)
        word_counts = defaultdict(int)

        # Indicator of document frequency.
        for uniq in unique_words:
            self.desc_dict[uniq] += 1

        # Indicator of term frequency.
        for word in tokens:
            word_counts[word] += 1

        # Processes the genre information of the current book.
        self.add_genre_info(genre, book.book_ix)
        # Adds the book to the necessary maps to make it
        # available in future time.
        self.desc_corpus[book.book_ix] = word_counts
        self.book_url_ix_map[book.book_url] = book.book_ix
        self.book_ix_url_map[str(book.book_ix)] = book.book_url
        self.book_is_added[str(book.book_ix)] = True

    def add_genre_info(self, genre_dict, book_ix):
        '''
        - Extracts the genre information by firstly making them
        lowercase than calculating their term frequency and
        document frequency. It checks for same genre as different
        instances in the book and accumulates them.
        '''
        lowered_dict = {}
        for genre, genreval in genre_dict.items():
            key = str(genre).lower()
            lowered_dict[key] = genreval

        uniq_genres = set(list(lowered_dict.keys()))
        genre_counts = defaultdict(int)

        for uniq in uniq_genres:
            self.genre_dict[uniq] += lowered_dict[uniq]

        for genre in lowered_dict.keys():
            genre_counts[genre] += lowered_dict[genre]

        self.genre_corpus[book_ix] = genre_counts

    def compose_a_query(self, selection, data):
        '''
        - Understands the query and returns the appropriate answer to
        the console. It supports single book query and multiple book
        query. For multiple book query, put the links into a file and
        type the filepath into the console. For further information,
        check the `help` function or README-REPORT.md file in project.
        '''
        if selection == "single":
            relevance_results, book_ix = self.query_a_book(data)
            precision, cumulative_match_ratio = self.calculate_query_statistics(book_ix, relevance_results)
            query_result_obj = { book_ix: { "relevances": relevance_results, "precision": precision, "avg_pre": cumulative_match_ratio } }
            self.display_query_results(query_result_obj)
        elif selection == "multiple":
            lines = read_book_urls(data)
            urls = [fix_url(url) for url in lines]
            query_results_obj = {}

            for url in urls:
                relevance_results, book_ix = self.query_a_book(url)
                precision, cumulative_match_ratio = self.calculate_query_statistics(book_ix, relevance_results)
                query_result_obj = { "relevances": relevance_results, "precision": precision, "avg_pre": cumulative_match_ratio }
                query_results_obj[book_ix] = query_result_obj

            self.display_query_results(query_results_obj)
        else:
            print("Invalid query form. Try again!")

    def query_a_book(self, book_url):
        '''
        - It queries a book by comparing its cosine similarity values
        with all other books in the library and returns the best
        matches to the console. It calculates query norm and document norm,
        then does dot product of a query and document pair, then finds the
        angle between them and maps them between 0 and 1.

        - for those variables which include "desc" or "d" string means it is
        related with the description of the book.
        - for those variables which include "genre" or "g" string means it is
        related with the genre information of the book.
        '''
        if book_url in set(self.book_url_ix_map.keys()):
            query_book_ix = str(self.book_url_ix_map[book_url])
            query_book_desc = self.desc_tf_idf[query_book_ix]
            query_book_genre = self.genre_tf_idf[query_book_ix]
        else:
            # Add clean url
            raw_book = self.download_a_book(fix_url(book_url))
            query_book_ix = raw_book.book_ix
            query_book_desc, query_book_genre = self.process_a_book(raw_book)

        scores = defaultdict(float)

        # Query vector norm part.
        qvec_desc_norm = 0.0
        qvec_genre_norm = 0.0
        for wval in query_book_desc.values():
            qvec_desc_norm += wval ** 2
        qvec_desc_norm = qvec_desc_norm ** 0.5

        for gval in query_book_genre.values():
            qvec_genre_norm += gval ** 2
        qvec_genre_norm = qvec_genre_norm ** 0.5

        for (dbook_ix, book_tf_idf), (gbook_ix, genre_tf_idf) in zip(self.desc_corpus.items(), self.genre_corpus.items()):

            if dbook_ix == query_book_ix:
                continue

            # Document vector norm part.
            dvec_desc_norm = 0
            dvec_genre_norm = 0
            for wval in book_tf_idf.values():
                dvec_desc_norm += wval ** 2
            dvec_desc_norm = dvec_desc_norm ** 0.5

            for gval in genre_tf_idf.values():
                dvec_genre_norm += gval ** 2
            dvec_genre_norm = dvec_genre_norm ** 0.5

            qword_set = set(query_book_desc.keys())
            dword_set = set(book_tf_idf.keys())
            # Finds the intersecting description between the query and the document
            desc_intersection = qword_set.intersection(dword_set)

            qgenre_set = set(query_book_genre.keys())
            dgenre_set = set(genre_tf_idf.keys())
            # Finds the intersecting genres between the query and the document
            genre_intersection = qgenre_set.intersection(dgenre_set)

            # Does dot product as in TF-IDF formula
            desc_dot_product = 0
            for iword in list(desc_intersection):
                desc_dot_product += query_book_desc[iword] * book_tf_idf[iword]

            # Does dot product as in TF-IDF formula
            genre_dot_product = 0
            for igenre in list(genre_intersection):
                genre_dot_product += query_book_genre[igenre] * genre_tf_idf[igenre]

            # Finds the angle between the query and the document for description
            try:
                book_desc_rel_score = desc_dot_product / ( qvec_desc_norm * dvec_desc_norm )
            except ZeroDivisionError:
                book_desc_rel_score = 0.0

            # Finds the angle between the query and the document for genre information
            try:
                book_genre_rel_score = genre_dot_product / ( qvec_genre_norm * dvec_genre_norm )
            except ZeroDivisionError:
                book_genre_rel_score = 0.0

            # Merges description and genre information values into one value
            scores[str(dbook_ix)] = (self.alpha * book_desc_rel_score) + ((1 - self.alpha) * book_genre_rel_score)

        # Deletes if the queried book itself is inside the documents.
        try:
            del scores[str(query_book_ix)]
        except:
            pass
        try:
            del scores[int(query_book_ix)]
        except:
            pass

        # Sorts the scores in ascending order.
        scores = dict(sorted(scores.items(), key=lambda score: score[1], reverse=True))
        self.last_query_scores = { "book_url": book_url, "scores": scores }
        # Returns the results and book index
        return list(scores.keys())[:self.relevant_count], query_book_ix

    def calculate_query_statistics(self, book_ix, relevance_results):
        '''
        - Calculates the precision and average precision values in the query
        as explained by the reference in the description.
        Reference link: http://sdsawtelle.github.io/blog/output/mean-average-precision-MAP-for-recommender-systems.html
        P.S.: Cumulative match ratio is written as average precision.
        '''
        rel_url_list = self.scan_book(book_ix)
        total_accurate_match = 0
        cumulative_match_ratio = 0.0
        for ix, _book_ix in enumerate(relevance_results):
            if str(_book_ix) == str(book_ix):
                continue
            book_url = str(self.book_ix_url_map[str(_book_ix)]).strip()
            if book_url in set(rel_url_list):
                total_accurate_match += 1
                cumulative_match_ratio += (total_accurate_match / (ix + 1))

        try:
            precision = total_accurate_match / len(relevance_results)
        except ZeroDivisionError:
            precision = 0.0
        try:
            cumulative_match_ratio = cumulative_match_ratio / total_accurate_match
        except ZeroDivisionError:
            cumulative_match_ratio = 0.0

        return precision, cumulative_match_ratio

    def remove_from_library(self, book_url):
        '''
        - Removes a book from the current library state with all the
        embedded information into the corpus and dictionary. For a
        stable state, the user should execute `recalibrate` command to
        get all TF-IDF values to be calculated again.

        P.S.: It may still have bugs, but it will be fixed as a future
        plan.
        '''
        if book_url not in set(self.book_url_ix_map.keys()):
            print("--> You cannot remove a book which is not added.")
            return

        book_ix = self.book_url_ix_map[book_url]
        if self.env_dev:
            fin = open(f"{self.test_dir}/{book_ix}.json", "r")
        else:
            fin = open(f"{self.prod_dir}/{book_ix}.json", "r")
        book_info = json.load(fin)
        fin.close()
        unique_tokens = set(processed_string(book_info["book_description"]))

        # remove dictionary occurences --> decreases document frequency
        for uniq in unique_tokens:
            self.desc_dict[uniq] = self.desc_dict[uniq] - 1

        # remove term frequency dictionary in this book
        del self.desc_corpus[str(book_ix)]
        del self.book_url_ix_map[book_url]
        del self.book_ix_url_map[str(book_ix)]
        del self.book_is_added[str(book_ix)]

        print(f"--> Book with ID: {book_ix} is deleted. System is out of sync.")
        print(f"--> You should recalibrate. Last system state date-time: {self.last_state}")

    def read_a_book(self, book_url):
        '''
        - Displays all information related to the book provided by Goodreads
        website. One can read a book's description and see its authors if one
        desires such a thing.
        '''
        if book_url not in set(self.book_url_ix_map.keys()):
            print("--> You cannot read a book which is not in the library.")
            return
        book_ix = self.book_url_ix_map[book_url]

        if self.env_dev:
            fin = open(f"{self.test_dir}/{book_ix}.json")
        else:
            fin = open(f"{self.prod_dir}/{book_ix}.json")

        book = json.load(fin)
        fin.close()
        book_title = book["book_title"]
        book_description = book["book_description"]
        book_authors = book["book_authors"]
        book_genre_info = book["book_genre_info"]

        print(f"Book Title:\t\t\t{book_title}")
        print(f"Book Authors:\t\t\t{book_authors}")
        print(f"Book Description:\t\t{book_description}")
        print(f"Book Genres:\t\t\t{book_genre_info}")

    def process_a_book(self, raw_book):
        '''
        - Calculates TF-IDF values of a book which is not added
        to the library beforehand. It brings agility to the application.

        P.S.: It may have minor bugs even though I do not think so.
        It will be verified in a future work.
        '''
        tokens = processed_string(raw_book.book_description)
        word_counts = defaultdict(int)
        genre_counts = defaultdict(int)
        desc_tf_idf_dict = defaultdict(float)
        genre_tf_idf_dict = defaultdict(float)
        for word in tokens:
            word_counts[word] += 1

        for word, freq in word_counts.items():
            try:
                desc_tf_idf_dict[word] = freq * math.log10(len(self.desc_corpus.keys())/self.desc_dict[word])
            except:
                desc_tf_idf_dict[word] = 0

        # Lowers the genre names
        lowered_dict = {}
        genre_dict = raw_book.book_genre_info
        for genre, genreval in genre_dict.items():
            lgenre = str(genre).lower()
            lowered_dict[lgenre] = genreval

        for genre, genreval in lowered_dict.items():
            genre_counts[genre] += genreval

        for genre, votes in genre_counts.items():
            try:
                genre_tf_idf_dict[genre] = votes * math.log10(len(self.genre_corpus.keys())/self.genre_dict[genre])
            except:
                genre_tf_idf_dict[genre] = 0

        return desc_tf_idf_dict, genre_tf_idf_dict

    def calculate_tf_idf(self):
        '''
        - Calculates TF-IDF values of all books provided at the
        initial state of the library. It calculates description
        and genre information TF-IDF values and registers them
        into several dictionaries to be used in further queries.
        '''
        N_DESC = len(self.desc_corpus.keys())
        N_GENRE = len(self.genre_corpus.keys())
        for book_ix, word_counts in self.desc_corpus.items():
            tf_idf_dict = defaultdict(float)
            for word, freq in word_counts.items():
                tf_idf_dict[word] = freq * math.log10(N_DESC/self.desc_dict[word])
            self.desc_tf_idf[str(book_ix)] = tf_idf_dict

        for book_ix, genre_counts in self.genre_corpus.items():
            tf_idf_dict = defaultdict(float)
            genre_sum = sum(genre_counts.values())
            for genre, votes in genre_counts.items():
                val = (votes/genre_sum) * math.log10(N_GENRE/self.genre_dict[genre])
                if val < 0:
                    val = val * -1
                tf_idf_dict[genre] = val

            self.genre_tf_idf[str(book_ix)] = tf_idf_dict

    def scan_book(self, book_ix):
        '''
        - Scans a book to extract recommendations for those who
        likes reading this book.
        '''
        if self.env_dev:
            fin = open(f"{self.test_dir}/{book_ix}.json", "r")
        else:
            fin = open(f"{self.prod_dir}/{book_ix}.json", "r")
        book = json.load(fin)
        fin.close()
        recommendations = book["book_recommendation_info"]
        rec_url_list = []
        for rec_obj in dict(recommendations).values():
            rec_url_list.append(rec_obj["url"])
        return rec_url_list

    def download_a_book(self, url):
        '''
        - Downloads the book from Goodreads website and creates
        a Book() instance to be used in further operations.
        Also, it exports the book information in a folder to be
        read by users of the Libraria.
        '''
        # Download the book page
        result = urllib.request.urlopen(url).read().decode("utf-8")
        # Increase the total number of books
        self.total_number_of_books += 1
        book_info = Book(result, self.total_number_of_books, url)
        if self.env_dev:
            book_info.dump_dict_json(self.test_dir)
        else:
            book_info.dump_dict_json(self.prod_dir)
        print(f"\rLast accessed url: {url}", end=" ")
        return book_info

    def display_query_results(self, query_result_obj):
        '''
        - Displays query results such as precision and average
        precision of the query made by a user along with a ranking
        of recommended books by my algorithms.
        '''
        for book_ix, query_result in dict(query_result_obj).items():
            precision = query_result["precision"]
            avg_pre = query_result["avg_pre"]
            relevances = query_result["relevances"]
            try:
                print(f"Query results for book url: {self.book_ix_url_map[book_ix]}")
            except:
                if self.env_dev:
                    fin = open(f"test_books/{book_ix}.json", "r")
                else:
                    fin = open(f"books/{book_ix}.json", "r")

                book = json.load(fin)
                fin.close()

                book_url = book["book_url"]
                print(f"Query results for book url: {book_url}")

            print(f"\tPrecision: {precision}")
            print(f"\tAverage Precision: {avg_pre}")
            print(f"\tRecommended book urls:")
            for rel in relevances:
                book_url = self.book_ix_url_map[rel]
                print(f"\t\tBook ID: {rel}\t Book Url: {book_url}")

    def display_last_query_scores(self):
        '''
        - Shows last query cosine similarity scores to the user.
        It is mainly used as development purposes, but any user can
        check the query values.
        '''
        print(self.last_query_scores)

class Book:
    def __init__(self, book_page_data, ix, url):
        '''
        - Book() class instance initialization function.
        Stores book index, url, title, authors, description, genre info
        and recommended books by Goodreads to be used in further operations.
        '''
        self.book_ix = ix
        self.book_url = url
        self.book_title = get_book_title(book_page_data)
        self.book_authors = get_book_authors(book_page_data)
        self.book_description = get_book_description(book_page_data)
        self.book_genre_info = get_genre_info(book_page_data)
        self.book_recommendation_info = get_recommended_books(book_page_data)

    def dump_dict(self):
        '''
        - Dumps the Book() instance to a dict object to be written into a file
        '''
        return {
            "book_id": self.book_ix,
            "book_url": self.book_url,
            "book_title": self.book_title,
            "book_authors": self.book_authors,
            "book_description": self.book_description,
            "book_genre_info": self.book_genre_info,
            "book_recommendation_info": self.book_recommendation_info
        }

    def dump_dict_json(self, path):
        '''
        - Exports a Book() instance to a JSON file to be used in further operations.
        '''
        book_dict = self.dump_dict()
        fjsonout = open(f"{path}/{self.book_ix}.json", "w")
        fjsonout.write(json.dumps(book_dict))
        fjsonout.flush()
        fjsonout.close()