import random
import sqlite3

class Account:

    @staticmethod
    def get_card_checksum(card_number: str) -> int:
        numbers_sum = 0
        for index in range(15):
            number = int(card_number[index])
            if index % 2 == 0:
                number *= 2
            if number > 9:
                number -= 9
            numbers_sum += number
        numbers_sum %= 10
        checksum = 10 - numbers_sum
        return checksum % 10

    @staticmethod
    def generate() -> "Account":
        random_account_number = random.randint(0, 999999999)
        number_15 = "400000" + "%09d" % random_account_number
        number = number_15 + str(Account.get_card_checksum(number_15))
        pin = "%04d" % random.randint(0, 9999)
        return Account(number, pin)

    def __init__(self, _number: str, _pin: str):
        self.number = _number
        self.pin = _pin

class Database:

    def __init__(self, file_name: str):
        self.conn = sqlite3.connect(file_name)
        self.conn.execute("CREATE TABLE IF NOT EXISTS card(id INTEGER, number TEXT, pin TEXT, balance INTEGER DEFAULT 0)")
        self.conn.commit()

    def insert_account(self, account: Account) -> None:
        self.conn.execute("INSERT INTO card VALUES (?, ?, ?, ?)",
                          (account.number, account.number, account.pin, 0))
        self.conn.commit()

    def check_login(self, number: str, pin: str) -> bool:
        c = self.conn.cursor()
        c.execute('SELECT number FROM card WHERE number = :number AND pin = :pin',
                  {"number": number, "pin": pin})
        return c.fetchone() is not None

    def get_balance(self, number: str) -> int:
        c = self.conn.cursor()
        c.execute('SELECT balance FROM card WHERE number = :number',
                  {"number": number})
        return c.fetchone()[0]

    def add_income(self, account: Account, income: int) -> None:
        self.conn.execute('UPDATE card SET balance = balance + :income WHERE id = :number',
                  {'income': income, 'number': account.number})
        self.conn.commit()

    def make_transfer(self, account: Account, account_to_transfer: str, amount: int) -> None:
        c = self.conn.cursor()
        c.execute('SELECT balance FROM card WHERE id = :number',
                  {"number": account.number})
        current_balance = c.fetchone()[0]

        if amount_to_transfer > current_balance:
            print("Not enough money!")
            return

        self.conn.execute('UPDATE card SET balance = balance - :amount WHERE id = :number',
                          {'amount': amount, 'number': account.number})
        self.conn.execute('UPDATE card SET balance = balance + :amount WHERE id = :number',
                          {'amount': amount, 'number': account_to_transfer})
        self.conn.commit()
        print("Success!")

    def account_exists(self, number: str) -> bool:
        c = self.conn.cursor()
        c.execute('SELECT number FROM card WHERE number = :number',
                  {"number": number})
        return c.fetchone() is not None

    def close_account(self, account: Account) -> None:
        self.conn.execute('DELETE FROM card WHERE number = :number',
                          {'number': account.number})
        self.conn.commit()
        print("The account has been closed!")

    def close(self) -> None:
        self.conn.commit()
        self.conn.close()

def login():  # -> Account
    print("Enter your card number:")
    input_card_number = input()

    print("Enter your PIN:")
    input_pin = input()

    if not db.check_login(input_card_number, input_pin):
        print("Wrong card number or PIN!")
        return None

    print("You have successfully logged in!")
    return Account(input_card_number, input_pin)

if __name__ == '__main__':
    db = Database('card.s3db')

    while True:
        print()
        print("1. Create an account")
        print("2. Log into account")
        print("0. Exit")

        option = int(input())

        if option == 1:
            account = Account.generate()
            db.insert_account(account)
            print("Your card has been created")
            print("Your card number:")
            print(account.number)
            print("Your card PIN:")
            print(account.pin)

        elif option == 2:
            account = login()
            if account is None:
                continue

            while True:
                print()
                print("1. Balance")
                print("2. Add income")
                print("3. Do transfer")
                print("4. Close account")
                print("5. Log out")
                print("0. Exit")

                logged_option = int(input())

                if logged_option == 1:
                    balance = db.get_balance(account.number)
                    print(f"Balance: {balance}")

                elif logged_option == 2:
                    print("Enter income:")
                    income_to_add = int(input())
                    db.add_income(account, income_to_add)
                    print("Income was added!")

                elif logged_option == 3:
                    print("Transfer")

                    print("Enter card number:")
                    account_to_transfer = input()
                    if account_to_transfer == account.number:
                        print("You can't transfer money to the same account!")
                        continue
                    if Account.get_card_checksum(account_to_transfer) != int(account_to_transfer[-1:]):
                        print("Probably you made mistake in the card number. Please try again!")
                        continue
                    if not db.account_exists(account_to_transfer):
                        print("Such a card does not exist.")
                        continue

                    print("Enter how much money you want to transfer:")
                    amount_to_transfer = int(input())
                    db.make_transfer(account, account_to_transfer, amount_to_transfer)

                elif logged_option == 4:
                    db.close_account(account)
                    break

                elif logged_option == 5:
                    print("You have successfully logged out!")
                    break

                elif logged_option == 0:
                    db.close()
                    exit(0)

        elif option == 0:
            print("Bye!")
            break

    db.close()


import math

def NormalizedScoreToLevel(score: float) -> float:
    return 0.5 * (math.sqrt(8*score + 1) + 1)

def ScoreToLevel(score: float, start: float, end: float, reversed: bool = False) -> float:
    range = end - start

    scalar = 45 / range

    normalizedScore = (score - start) * scalar

    #Use reversed when bigger jumps are easier intially and the last few points are difficult.
    if reversed:
        level = NormalizedScoreToLevel(45-normalizedScore)
        level = 11 - level
    else:
        level = NormalizedScoreToLevel(normalizedScore)

    return level

def NextGoal(level : float, start: float, end: float, reversed: bool = False) -> float:
    nextLevel = math.ceil(level + 0.001)

    return LevelToScore(nextLevel, start, end, reversed)

def LevelToScore(level : float, start: float, end: float, reversed: bool = False) -> float:

    range = end - start

    scalar =  45 / range

    if reversed:
        level = 11 - level
        target = level * (level - 1) / 2
        return end - (target / scalar)
    else:
        target = level * (level - 1) / 2
        return (target / scalar) + start


from importlib.abc import MetaPathFinder
from _frozen_importlib_external import PathFinder
import sys
import pathlib
import warnings
from typing import List, Any, Union, Iterable

PYODIDE = True

try:
    from js import XMLHttpRequest
except:
    warnings.warn("Pyodide not found, running in native mode")
    import requests

    PYODIDE = False

class PyFinder(MetaPathFinder):
    """Meta path finder for Pyodide which supports importing external Python files"""

    def __init__(
        self,
        import_path: Union[str, List[str]] = "",
        download_path: str = "",
        modules: Iterable[str] = None,
    ):
        self.import_path = self._prepare_import_path(import_path)
        self.download_path = self._to_abspath(download_path)
        self.modules = set(modules) if modules is not None else None

    def register(self):
        """
        Register the PyFinder to sys.meta_path
        """
        if not self._registered():
            sys.meta_path.append(self)

    def unregister(self):
        """
        Unregister the PyFinder from sys.meta_path
        """
        if self._registered():
            sys.meta_path.remove(self)

    def _registered(self):
        """
        Check whether the PyFinder is registered in sys.meta_path
        """
        return self in sys.meta_path

    def __enter__(self):
        self.register()

    def __exit__(self, type, value, traceback):
        self.unregister()

    def _prepare_import_path(self, paths: Union[str, List[str]]):
        if isinstance(paths, str):
            return [paths.rstrip("/") + "/"]
        elif isinstance(paths, list):
            return [path.rstrip("/") + "/" for path in paths]

    def _to_abspath(self, path: str):
        return pathlib.Path(path).resolve()

    def add_module(self, module: Union[str, List[str]]):
        """
        Add new module(s) to the whitelist which PyFinder import hook can import
        """
        if self.modules is None:
            self.modules = set()

        if isinstance(module, str):
            self.modules.add(module)
        else:
            self.modules.update(module)

    def available_modules(self):
        """
        Get the list of whitelisted modules which PyFinder import hook can import
        """
        return self.modules

    @staticmethod
    def invalidate_caches():
        """Call the invalidate_caches() method on all path entry finders
        stored in sys.path_importer_caches (where implemented)."""
        for name, finder in list(sys.path_importer_cache.items()):
            if finder is None:
                del sys.path_importer_cache[name]
            elif hasattr(finder, "invalidate_caches"):
                finder.invalidate_caches()

    def find_spec(self, fullname: str, path: List[str], target: Any = None):
        # Should find python packages according to PEP-420.
        raise NotImplementedError

class PyHTTPFinder(PyFinder):
    """Meta path finder for importing python files through HTTP(s)"""

    def __init__(
        self,
        import_path: Union[str, List[str]] = "",
        download_path: str = "",
        modules: Iterable[str] = None,
    ):
        super().__init__(import_path, download_path, modules)

    def _get(self, url: str):
        """
        Download python script through HTTP(s)
        """
        if PYODIDE:
            req = XMLHttpRequest.new()
            req.open("GET", url, False)
            req.send(None)
            return (req.status, req.responseText)
        else:
            # This enables testing this module outside of Pyodide
            resp = requests.get(url)
            return (resp.status_code, resp.text)

    def find_spec(self, fullname: str, path: List[str], target: Any = None):
        # Search order:
        # 1. <directory>/foo/__init__.py
        # 2. <directory>/foo.py
        # (-) Not supported: binary formats (pyc, pyd, so)
        # (-) Not supported: Namespace packages without __init__.py

        # module1.module2 ==> module1/module2
        modules = fullname.split(".")
        if self.modules is not None and modules[0] not in self.modules:
            return None

        fullname_aspath = fullname.replace(".", "/")

        import_subpaths = [
            f"{fullname_aspath}/__init__.py",  # Regular package
            f"{fullname_aspath}.py",  # Module package
        ]

        for base_path in self.import_path:
            for import_subpath in import_subpaths:
                status_code, content = self._get(f"{base_path}{import_subpath}")
                if status_code != 200:
                    continue

                package_path = self.download_path / import_subpath
                package_path.parent.mkdir(parents=True, exist_ok=True)
                with package_path.open("w") as f:
                    f.write(content)

                # We need to invalidate caches since PathFinder have already failed once and it is cached.
                self.invalidate_caches()
                return PathFinder.find_spec(fullname, path, target)

        return None


import requests
import streamlit as st
nome_do_filme = st.text_input('Choose Your Movie: ')
if st.button('Consultar Filmes'):

    #é recomendado procurar filmes populares. Link fonte não possui um banco de dados extenso
    link = 'https://www.omdbapi.com/?apikey=90c1591a&t=' + nome_do_filme
    dvd = requests.get(link)
    filme = dvd.json()
    # utilizei a chave response para verificar se o filme estava na API
    if filme['Response'] == 'True':
        actors = filme['Actors'].split(', ')
        first_actor = actors[0]
        st.write("Atores: ", filme['Actors'])
        st.write("Gênero: ", filme['Genre'])
        st.write("Classificação Indicativa: ", filme['Rated'])
        st.write("\n")

        link2 = 'https://www.omdbapi.com/?apikey=90c1591a&s=' + first_actor  # Search for movies by first actor's name
        dvd2 = requests.get(link2)
        search_results = dvd2.json()

        if search_results['Response'] == 'True' and search_results['totalResults'] != '0':
            movies = search_results['Search']  # Limit the number of movies to 3
            st.write("Primeiro ator: ", first_actor)
            st.write("Outros filmes dele:")
            for movie in movies:
               st.write(movie['Title'])
        else:
          st.write("Não foram encontrados outros filmes para este ator.")


# -*- coding: utf-8 -*-
import sys
from os import path
import logging
import re
import numpy as np
from tqdm import tqdm

sys.path.append('./src/')
from utils import argparser as parser
from utils import utils

def get_covariances(embs_full, min_samples):
    covs, var = {}, {}
    for word, embs in embs_full.items():
        if embs.shape[0] > min_samples and len(word) > 1:
            covs[word] = {
                'covariance': np.cov(embs, rowvar=False),
                'n_samples': embs.shape[0]
            }
            var[word] = {
                'variance': np.var(embs, axis=0, ddof=1),
                'n_samples': embs.shape[0]
            }
    return covs, var

def save_covs(covs, file_path, file_key):
    tqdm.write('\tSaving file')
    filename = path.join(file_path, 'covs_%s.pickle.bz2' % (file_key))
    utils.write_pickle(filename, covs)

def get_filekeys(filename, data_type):
    base_name = '%s_%s.pickle.bz2' % (data_type, r'(.+?)')
    filename = filename.split('/')[-1]
    m = re.match(base_name, filename)
    return m.group(1)

def main():
    args = parser.parse_args()
    logging.info(args)

    var = {}
    filenames = utils.get_filenames(args.embeddings_merged_path)
    for i, filename in tqdm(list(enumerate(filenames))):
        tqdm.write('%d/%d Reading file: %s. Length of vars: %d' %
                   (i + 1, len(filenames), filename, len(var)))
        embs = utils.read_pickle(filename)
        covs, var_temp = get_covariances(embs, args.min_samples)
        var = dict(var, **var_temp)

        if len(covs) > 0:
            file_key = get_filekeys(filename, 'embs')
            save_covs(covs, args.embeddings_covariance_path, file_key)

    utils.write_pickle(args.embeddings_variance_file, var)

if __name__ == '__main__':
    main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Author: gaopeng
# @Date:   2018-01-22 16:52:53
# @Last modified by:   gaopeng
# @Last modified time: 2018-01-24 15:50:33

import logging; logging.basicConfig(level=logging.INFO)

import os, time
from datetime import datetime

from transwarp import db
from transwarp.web import WSGIApplication, Jinja2TemplateEngine

from config import configs

def datetime_filter(t):
    delta = int(time.time() - t)
    if delta < 60:
        return u'1分钟前'
    if delta < 3600:
        return u'%s分钟前' % (delta // 60)
    if delta < 86400:
        return u'%s小时前' % (delta // 3600)
    if delta < 604800:
        return u'%s天前' % (delta // 86400)
    dt = datetime.fromtimestamp(t)
    return u'%s年%s月%s日' % (dt.year, dt.month, dt.day)

db.create_engine(**configs.db)
wsgi = WSGIApplication(os.path.dirname(os.path.abspath(__file__)))
template_engine = Jinja2TemplateEngine(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'templates'))

wsgi.template_engine = template_engine
# 把filter添加到jinjia2，filter名称为datetime，filter本身是一个函数对象:
template_engine.add_filter('datetime', datetime_filter)

import urls

wsgi.add_interceptor(urls.user_interceptor)
wsgi.add_interceptor(urls.manage_interceptor)
wsgi.add_module(urls)

if __name__ == '__main__':
    wsgi.run(10088, host='0.0.0.0')
else:
    application = wsgi.get_wsgi_application()


from main import app, db
from views import Form, feature_flag
from models.ticket import TicketType
from models.cfp import (
    TalkProposal, WorkshopProposal,
    InstallationProposal, ProposalDiversity,
)

from flask import (
    render_template, redirect, request,
    url_for, abort,
)
from flask.ext.login import current_user

from wtforms.validators import Required, Email
from wtforms import (
    BooleanField, TextField,
    FormField, TextAreaField, SelectField,
)

class DiversityForm(Form):
    age = TextField('Age')
    gender = TextField('Gender')
    ethnicity = TextField('Ethnicity')

class ProposalForm(Form):
    name = TextField("Name", [Required()])
    email = TextField("Email", [Email(), Required()])
    title = TextField("Title", [Required()])
    description = TextAreaField("Description", [Required()])
    # days = SelectMultipleField("I can only attend on some days",
    #                            choices=[('fri', 'Friday'),
    #                                     ('sat', 'Saturday'),
    #                                     ('sun', 'Sunday'),
    #                                     ])
    need_finance = BooleanField("I can't afford to buy a ticket without financial support")

    diversity = FormField(DiversityForm)

class TalkProposalForm(ProposalForm):
    type = 'talk'
    length = SelectField("Duration", default='30',
                         choices=[('< 10 mins', "Shorter than 10 minutes"),
                                  ('10 mins', "10 minutes"),
                                  ('30 mins', "30 minutes"),
                                  ('45 mins', "45 minutes"),
                                  ('> 45 mins', "Longer than 45 minutes"),
                                  ])
    experience = SelectField("Have you given a talk before?",
                             choices=[('none', "It's my first time"),
                                      ('some', "I've talked before"),
                                      ('repeat', "I've given this talk before"),
                                      ])
    one_day = BooleanField("I can only attend for the day I give my talk")

class WorkshopProposalForm(ProposalForm):
    type = 'workshop'
    length = TextField("Duration", [Required()])
    attendees = TextField("Attendees", [Required()])
    one_day = BooleanField("I can only attend for the day I give my workshop")

class InstallationProposalForm(ProposalForm):
    type = 'installation'
    size = TextField("Physical size", [Required()])

@app.route('/cfp')
@app.route('/cfp/<string:cfp_type>', methods=['GET', 'POST'])
@feature_flag('CFP')
def cfp(cfp_type='talk'):
    if cfp_type not in ['talk', 'workshop', 'installation']:
        abort(404)

    forms = [TalkProposalForm(), WorkshopProposalForm(), InstallationProposalForm()]
    (form,) = [f for f in forms if f.type == cfp_type]

    if request.method == 'POST':
        app.logger.info('Checking %s proposal for %s (%s)', cfp_type, form.name.data, form.email.data)

    if form.validate_on_submit():
        if cfp_type == 'talk':
            cfp = TalkProposal()
            cfp.length = form.length.data
            cfp.experience = form.experience.data
            cfp.one_day = form.one_day.data
        elif cfp_type == 'workshop':
            cfp = WorkshopProposal()
            cfp.length = form.length.data
            cfp.attendees = form.attendees.data
            cfp.one_day = form.one_day.data
        elif cfp_type == 'installation':
            cfp = InstallationProposal()
            cfp.size = form.size.data

        cfp.name = form.name.data
        cfp.email = form.email.data
        cfp.title = form.title.data
        cfp.description = form.description.data
        cfp.need_finance = form.need_finance.data

        cfp.diversity = ProposalDiversity()
        cfp.diversity.age = form.diversity.age.data
        cfp.diversity.gender = form.diversity.gender.data
        cfp.diversity.ethnicity = form.diversity.ethnicity.data

        db.session.add(cfp)
        db.session.commit()

        return redirect(url_for('cfp_complete'))

    if current_user.is_authenticated():
        form.name.data = current_user.name
        form.email.data = current_user.email

    full_price = TicketType.query.get('full').get_price('GBP')

    return render_template('cfp.html', full_price=full_price,
        forms=forms, active_cfp_type=cfp_type, has_errors=bool(form.errors))

@app.route('/cfp/complete')
@feature_flag('CFP')
def cfp_complete():
    return render_template('cfp_complete.html')


#!/usr/bin/python3
""" This modules has a script that adds all arguments provided to a\
python list, writes them on a file and svaes them there """
import sys
save_to_json_file = __import__('5-save_to_json_file').save_to_json_file
load_from_json_file = __import__('6-load_from_json_file').load_from_json_file

filename = 'add_item.json'

try:
    args = load_from_json_file(filename)
except Exception:
    args = []
if len(sys.argv) - 1:
    idx = 1
    while idx < len(sys.argv):
        args.append(sys.argv[idx])
        idx += 1
save_to_json_file(args, filename)


# -*- coding: utf-8 -*-
"""
Created on Mon Aug 17 19:21:08 2020

@author: bmcma
"""

import rnn
import numpy as np
import torch
import matplotlib.pyplot as plt
import svcca.cca_core as cca_core
from sklearn.manifold import MDS
from sklearn.cluster import KMeans
from sklearn.mixture import GaussianMixture
from sklearn.metrics import silhouette_score
import pdb
import argparse
from FP_Analysis import ComputeDistance

# TODO: refactor code as class
# class NetworkDistances:
#     # public:
#     def getDistances():
#         self._getActivations()
#     # private:
#     def loadRNNS():  # loads RNNs to be analyzed
#         pass
#     def _getActivations():  # gets the activations of all RNNs
#         pass

def getNumSVs(singularValues):
    sv_squares = singularValues**2
    svs_needed = 0
    explained_var = 0
    sv_square_totals = np.sum(sv_squares)
    while (explained_var < 0.95):
        explained_var += (singularValues[svs_needed]**2 / sv_square_totals)
        svs_needed+=1
    return svs_needed

def get_rdm_inputs(inputSize):
    # returns inputs for the RDM task
    #assert False, "you are using RDM, use N=1 network instead"
    NUM_INPUT_CONDITIONS = 10
    static_inputs = np.linspace(-0.1857, 0.1857, NUM_INPUT_CONDITIONS).reshape(1, NUM_INPUT_CONDITIONS)
    static_inputs = np.matmul(np.ones((750, 1)), static_inputs)
    static_inputs = torch.tensor(static_inputs).float()
    static_inputs = torch.unsqueeze(static_inputs.t(), 1)
    return static_inputs

def get_static_inputs(inputSize):
    # will get inputs for the N = 1 context task
    # N_INPUTS = 120   # needs to be a multiple of all possible N
    N_INPUTS_PER_CONTEXT = 10
    N_CONTEXTS = int(inputSize / 2)
    #N_INPUTS_PER_CONTEXT = int(N_INPUTS / N_CONTEXTS)
    MIN_INPUT = -0.1857
    MAX_INPUT = 0.1857
    static_inpt_arr = np.zeros((N_INPUTS_PER_CONTEXT*N_CONTEXTS, inputSize))
    #static_inpt_arr = np.zeros((N_INPUTS_PER_CONTEXT*N_CONTEXTS, inputSize))
    # loop over contexts
    for context in range(N_CONTEXTS):
       strt_ix = context*N_INPUTS_PER_CONTEXT
       end_ix = (context+1)*N_INPUTS_PER_CONTEXT
       static_inpt_arr[strt_ix:end_ix, context] = np.linspace(MIN_INPUT, MAX_INPUT, N_INPUTS_PER_CONTEXT)  # set inputs
       static_inpt_arr[strt_ix:end_ix, context+N_CONTEXTS] = 1  # set GO signal
    # static_inpt_arr now filled in with shape (20, 8)
    static_inpt_arr = np.expand_dims(static_inpt_arr, -1)  # (20, 8, 1)
    assert static_inpt_arr.shape == (N_INPUTS_PER_CONTEXT*N_CONTEXTS, inputSize, 1)
    inpts = np.matmul(static_inpt_arr, np.ones((1, 750))) # (20, 8, 750)
    # feed batch of 20 inpts into network to get ativations
    # cast static inputs to PyTorch Tensor
    static_inputs = torch.tensor(inpts).float() # 20, 8, 750
    return static_inputs

def getActivations(rnn_model):
    '''generates inputs for SVCCA'''
    switcher = {
            1:get_rdm_inputs,
            2:get_static_inputs,
            4:get_static_inputs,
            6:get_static_inputs,
            8:get_static_inputs,
            10:get_static_inputs,
            12:get_static_inputs
        }
    static_inputs = switcher[rnn_model._inputSize](rnn_model._inputSize)
    _, activations = rnn_model.feed(static_inputs, return_hidden=True)
    if rnn_model._useHeb:
        heb_mask = np.ones(50).astype(int)
        heb_mask[1] = 0; heb_mask[10] = 0; heb_mask[11] = 0
        activations = activations[:, np.where(heb_mask==1)[0], :]
        activations = np.swapaxes(activations, 0, 1).reshape(rnn_model._hiddenSize-3, -1)
        return activations #finalActivations    # 50 x 500  ----> numHidden x numInputs
    #finalActivations = activations[-1,:,:]       # keeps activation at end of trial only
    activations = np.swapaxes(activations, 0, 1).reshape(rnn_model._hiddenSize, -1)
    return activations #finalActivations    # 50 x 500  ----> numHidden x numInputs

# get a file of models to analyze
parser = argparse.ArgumentParser(description="Clusters RNNs by SVCCA")
parser.add_argument("fname", help="name of file containing RNNs to analyze")
parser.add_argument("-reload", action="store_true", default=False)
args = parser.parse_args()
file_of_rnns = open("models/"+args.fname, 'r')
models = [(line.strip()).split() for line in file_of_rnns]
file_of_rnns.close()

modelActivations = []
NUM_DIMENSIONS = 2

#models = [bptt, ga]
numModelsOfType = {}
count = 0
for i in range(len(models)):
    numModelsOfType[models[i][0]] = len(models[i])-1
    count += len(models[i]) - 1
numModelsOfType["total"] = count

if args.reload:
    distances = np.zeros((numModelsOfType["total"], numModelsOfType["total"]))
    for modelType in models:
        for count, model in enumerate(modelType):
            if (count == 0):
                print("model name:", model)
            else:
                print("name:", model)
                rnn_inst = rnn.loadRNN("models/"+model)
                activations = getActivations(rnn_inst)
                modelActivations.append(activations)

    # compute the distance matrix
    print("computing distance matrix ...\n")
    for i in range(len(modelActivations)):
        print("    Computing row", int(i))
        for j in range(i, len(modelActivations)):   # ~2X speedup using symmetry
            activationsI = modelActivations[i]
            activationsJ = modelActivations[j]
            activationsI -= np.mean(activationsI, axis=1, keepdims=True)
            activationsJ -= np.mean(activationsJ, axis=1, keepdims=True)

            # take top 20 singular values
            U1, s1, V1 = np.linalg.svd(activationsI, full_matrices=False)
            U2, s2, V2 = np.linalg.svd(activationsJ, full_matrices=False)

            #print("Number of singular values needed to explain 95% of variance:", getNumSVs(s1))
            NUM_DIMENSIONS = getNumSVs(s1)
            svacts1 = np.dot(s1[:NUM_DIMENSIONS]*np.eye(NUM_DIMENSIONS), V1[:NUM_DIMENSIONS])
            NUM_DIMENSIONS = getNumSVs(s2)
            svacts2 = np.dot(s2[:NUM_DIMENSIONS]*np.eye(NUM_DIMENSIONS), V2[:NUM_DIMENSIONS])

            if svacts1.shape != svacts2.shape:
                print("SVACTS1:", svacts1.shape)
                print("SVACTS2:", svacts2.shape)
            svcca_results = cca_core.get_cca_similarity(svacts1, svacts2, epsilon=1e-10, verbose=False)
            distances[i, j] = 1 - np.mean(svcca_results["cca_coef1"])
            distances[j, i] = distances[i, j]
            # distances are shape (numRNNs, numRNNs)

    # save the distances
    np.save("models/" + args.fname[:-4] + "_distances", distances)
else:
    distances = np.load("models/" + args.fname[:-4] + "_distances.npy")
plt.imshow(distances)
plt.clim(0, 1.0/3)
plt.colorbar()
plt.title("SVCCA Distances Between Networks")

clustering_algorithm = MDS()
clustered_data = clustering_algorithm.fit_transform(distances)

plt.show()

def Method1(rnn_distances, labels, class_counts):
    '''
    Model Agnostic Method for measuring how clustered Learning Rules are
    1.) Find the mean for each learning rule
    2.) Compute distance of all points for a given learning rule to the mean
        for that learning rule (d_lr)
    3.) Compute the distance of all points from other learning rules to the
        mean for the current learning rule (d_other)
    4.) Compute the ratio of d_lr / d_other
    '''
    N_RNNS = len(rnn_distances)
    # compute the mean/centroid for each learning rule
    centroid_positions = {0: np.zeros((N_RNNS)), 1: np.zeros((N_RNNS)), 2: np.zeros((N_RNNS)), 3: np.zeros((N_RNNS))}
    for rnn_ix in range(N_RNNS):  # loop over all RNNs
        centroid_positions[labels[rnn_ix]] += rnn_distances[rnn_ix]
    # now we must normalize the means
    for class_key in centroid_positions:
        centroid_positions[class_key] /= class_counts[class_key]
    # centroid_positions now contains the centroid for each learning rule

    # loop through all RNNs and get distance from centroids
    ooc_distances = {0:[], 1:[], 2:[], 3:[]}
    within_class_distances = {0:[], 1:[], 2:[], 3:[]}
    for rnn_ix in range(N_RNNS):
        # compute within class distance
        for class_key in centroid_positions:
            if class_key == labels[rnn_ix]:
                # compute within class distance\
                within_class_distances[class_key].append(ComputeDistance(centroid_positions[class_key], rnn_distances[rnn_ix]))
            else:
                # compute out of class distance
                ooc_distances[class_key].append(ComputeDistance(centroid_positions[class_key], rnn_distances[rnn_ix]))

    # get the mean of each list in both dictionaries and get the ratio
    ratios = []
    for class_key in within_class_distances:
        ratios.append(np.mean(within_class_distances[class_key])/np.mean(ooc_distances[class_key]))
    return ratios

def Method2(rnn_distances, labels, class_counts):
    '''
    Model Agnostic Method for measuring how clustered Learning Rules are
    1.) Compute average distance for any two points in the same learning rule
        (d_same)
    2.) Compute average distance between any two points (d_all)
    3.) Ratio of d_same / d_all
    '''
    N_RNNS = len(rnn_distances)

    # loop through all RNNs and get distance from all other RNNs
    ooc_distances = {0:[], 1:[], 2:[], 3:[]}
    within_class_distances = {0:[], 1:[], 2:[], 3:[]}
    for rnn_i in range(N_RNNS):
        for rnn_j in range(rnn_i+1, N_RNNS):
            # compute distance between RNN_i and RNN_j
            d = ComputeDistance(rnn_distances[rnn_i], rnn_distances[rnn_j])
            if labels[rnn_i] == labels[rnn_j]:  # update within class
                within_class_distances[labels[rnn_i]].append(d)
            else:  # update out-of-class
                ooc_distances[labels[rnn_i]].append(d)
                ooc_distances[labels[rnn_j]].append(d)
    # get the mean of each list in both dictionaries and get the ratio
    ratios = []
    for class_key in within_class_distances:
        ratios.append(np.mean(within_class_distances[class_key])/np.mean(ooc_distances[class_key]))
    return ratios

def Purity(rnn_distances, predicted, labels):
    '''
    Computes purity of clusters found with SVCCA

    Parameters
    ----------
    rnn_distances : obj
        model will be saved as a .pt file with this name in the /models/ directory.

    predicted : list<int>
        list of predicted class for each rnn

    labels : list<int>
        list of true class for each rnn

    Returns
    -------
    None.

    for each cluster,
        count the number of data points from the most common learning rule
        sum this accross all clusters
    divide the running sum by the total number of data points

    '''
    N_CLASSES = 4  # total number of rnn classes
    N_CLUSTERS = 4 # total number of rnn clusters (should be same as classes)
    N_RNNS = len(distances)  # total number of rnns clustered
    # dictionary counts the number of points belonging to each class
    # for each cluster. For example, class_counts[0][1] = 3 would
    # indicate that there were three points belonging to class 1 in
    # cluster 3. More generally,
    #    class_counts[cluster_id][class_id] = num_points_in_class_id_and_cluster_id
    #
    class_counts = {0:{0:0, 1:0, 2:0, 3:0},
                    1:{0:0, 1:0, 2:0, 3:0},
                    2:{0:0, 1:0, 2:0, 3:0},
                    3:{0:0, 1:0, 2:0, 3:0}}
    for ix in range(N_RNNS):  # loop through each RNN
        cluster_id = predicted[ix]  # cluster predicted by model
        class_id = labels[ix]       # true class of rnn
        # increment number of this class contained in this cluster
        class_counts[cluster_id][class_id] += 1

    # now we must sum the total number of points for each cluster belonging to the maximum
    # represented class
    max_per_class = np.zeros((N_CLUSTERS, 1))
    for cluster_id in range(N_CLASSES):
        # get the max in this dictionary
        max_pts_found = -1
        for key in class_counts[cluster_id]:
            if class_counts[cluster_id][key] > max_pts_found:  # current class contains the most rnns
                max_pts_found = class_counts[cluster_id][key]
        max_per_class[cluster_id] = max_pts_found
    # max_per_class now contains the number of points for the most represented class in
    # each cluster
    purity = np.sum(max_per_class, axis=0) / N_RNNS
    return purity

def getTrueLabelsHelper(numModelsOfType):
    '''
    numModelsOfType is a dictionary where
    each key is a model type and each value
    is the number of models of that type
    '''
    labels = []  # holds the ground truth labels
    class_counts = [0 for i in range(len(numModelsOfType)-1)]
    for class_id, key in enumerate(numModelsOfType):
        if key=="total":
            continue   # don't use totals
        curr_class_count = 0
        for j in range(numModelsOfType[key]):
            labels.append(class_id)
            curr_class_count += 1  # number of rnns in this class
        class_counts[class_id] = curr_class_count
    # note the labels in class_id don't have to
    # correspond with the labels used by the clustering
    # algorithm since we only consider purity and not
    # that the labels match
    return labels, class_counts

labels, class_counts = getTrueLabelsHelper(numModelsOfType)

if len(numModelsOfType) != 5 and len(numModelsOfType) != 3:
    N_colors = ['r', 'orange', 'g', 'b', 'indigo', 'violet']  # use when plotting against noise levels
else:
    N_colors = ['blue', 'orange', 'green', 'red']              # use when plotting the 4 learning rules
plt.figure(11)  # Scatter plot
count = 0     # keeps track of how many models plotted so far
ci = 0
legends = []  # holds model names to be used in legend
for modelType in numModelsOfType:
    if modelType == "total": # no plotting for total since it isn't a valid model type
        continue
    value = numModelsOfType[modelType]
    if modelType[0] == 'B':

        plt.scatter(clustered_data[count:count+value,0], clustered_data[count:count+value,1], c=N_colors[ci])
    else:

        plt.scatter(clustered_data[count:count+value,0], clustered_data[count:count+value,1], c=N_colors[ci])
    i+=1
    ci+=1
    count += value  # increment the number of models plotted
    legends.append(modelType) # add this model type name to the legend

plt.legend(legends)
plt.title("Sillhouette Score:" + str(silhouette_score(distances, labels)))

plt.figure()
x = np.array([ 1, 2, 3, 4, 5, 6])
y = np.array([0.25, 6.2e-4, 0.012, -0.013, -1.7e-5, -0.015])

m, b = np.polyfit(x, y, 1)
plt.plot(x, y, 'o')
plt.plot(x, m*x + b)
plt.ylim([-0.1, 0.5])


"""
    Main
"""

import json

from Modules.smb_client import SmbClient
from Modules.archive import Archive
from Modules.mail import Mail

CONFIG = "config.json"

client = SmbClient(CONFIG)
archive = Archive(CONFIG)
mail = Mail(CONFIG)

with open(CONFIG, encoding="utf8") as conf:
    data = json.load(conf)
    mail_json = data["mail"]

def main():
    """
    Main script of this project
    """
    body = ""
    subject=""

    archive.clear_log_file()
    archive.download_file()
    archive.extract_zip()
    archive.check_file()
    archive.create_tgz()
    archive.clean_zip()
    archive.clean_file()
    archive.send_tgz(client)
    archive.clean_tgz()
    archive.clean_server(client)
    body = archive.read_log_file()
    subject = archive.read_log_file_first_line()

    if mail_json["send_mode"] == "on":
        if mail_json["attach_mode"] == "on":
            mail.send_email_ssl_with_log(body, subject)
        else:
            mail.send_email_ssl(body, subject)

    archive.delete_log_file()

if __name__ == "__main__":
    main()


from typing import *

a = [1, 2, 3]                   # list of ints
a = [11.0, 12, 13.0]            # wrong, list of ints and floats
a = (4, 5, 6)                   # wrong, tuple of ints rather than list of ints
a = {4, 5, 6}                   # wrong, set of ints rather than list of ints
a = [7, 8, 9, 10, 11]           # right, list ints

b = [1.0, 2, 3]                 # list of ints and floats
b = [1, 2.0, 3.0]               # right, list of ints and floats
b = [1.0, 2.0, 'three']         # wrong, list of ints, floats and strings

c = [1.1, 2.2, 3.3]             # list of floats
c = [1, 2, 3]                   # right, list of integers, which are specializations of floats
c = [4, 5.5, 6]                 # right, list of integers and floats, which are specializations of integers

d = (1, 2)                      # tuple of 2 ints
d = (3, 4, 5)                   # wrong, tuple of 3 ints

e: List [int] = []              # list of ints, explicitly typed since it is empty
e = [1, 2.5]                    # wrong, float is no specialization of int
e = [3, 4, 5]                   # right, list of ints

f = [[1]]                       # list of lists of ints
f = [[1, 2], [3, 4]]            # right, list of list of ints
f = [[1.5, 2.5], [3.5, 4.5]]    # wrong, float is no specialization of int


from flask import request
from flask import Blueprint
import pdb
from . import db
import face_recognition
import pickle

bpAñadir = Blueprint('anadir', __name__)
@bpAñadir.route('/add', methods = ['POST'])
def add():

    if request.method == 'POST':
        # Desde la parte de params de postman, usamos args
        # id = request.args.get('idfoto')

        if 'file' not in request.files or 'iduser' not in request.form or 'idfoto' not in request.form:
            return db.error_400, 400

        idUser = request.form.get("iduser")
        idFoto = request.form.get("idfoto")
        file = request.files['file']

        if not db.allowed_file(file.filename):
            return db.error_415, 415

        #Cargamos la imagen en un array numpy
        img = face_recognition.load_image_file(file)

        #Puede ser que no se detecten caras en la imagen
        try:
            img_encoding = face_recognition.face_encodings(img)[0]
        except IndexError as e:
            print(e)
            return db.error_400_1, 400

        cur = db.get_db().cursor()

        fila = cur.execute("SELECT * FROM users WHERE idUser=" + idUser + " AND idFoto=" + idFoto).fetchall()
        if len(fila) != 0:
            cur.execute("DELETE FROM users WHERE  idUser=" + idUser + " AND idFoto=" + idFoto)
            cur.execute("INSERT INTO users (idUser, idFoto, stringFoto) VALUES (?,?,?)",
                    (idUser, idFoto, pickle.dumps(img_encoding)))
            db.get_db().commit()
            cur.close()
            return db.update, 200
        else:
            cur.execute("INSERT INTO users (idUser, idFoto, stringFoto) VALUES (?,?,?)",
                    (idUser, idFoto, pickle.dumps(img_encoding)))
            db.get_db().commit()
            cur.close()
            return db.add, 201


# Go to root of PyNXBot
import sys
sys.path.append('../')
from lookups import PKMString
from nxbot import SWSHBot
from structure import EncounterNest8Archive
from structure import NestHoleReward8Archive

pmtext = PKMString()
buf = bytearray(open('../resources/bytes/local_raid','rb').read())
encounter = EncounterNest8Archive.GetRootAsEncounterNest8Archive(buf,0)

buf = bytearray(open('../resources/bytes/local_drop','rb').read())
drop = NestHoleReward8Archive.GetRootAsNestHoleReward8Archive(buf,0)

buf = bytearray(open('../resources/bytes/local_bonus','rb').read())
bonus = NestHoleReward8Archive.GetRootAsNestHoleReward8Archive(buf,0)

if encounter.TablesIsNone():
	print('Wrong offset!')
else:
	for ii in range(encounter.TablesLength()):
		table = encounter.Tables(ii);
		print(f"Table ID:0x{table.TableID():X}")
		print(f"Game Version:{table.GameVersion()}")
		for jj in range(table.EntriesLength()):
			entry = table.Entries(jj)
			msg = f"{entry.EntryIndex()}: {'G-' if entry.IsGigantamax() else ''}{pmtext.species[entry.Species()]}{('-' + str(entry.AltForm())) if entry.AltForm() > 0 else ''}"
			msg = f"{msg:25}\t"
			if entry.Ability() == 4:
				msg +=f"HA allowed\t"
			elif entry.Ability() == 3:
				msg +=f"no HA\t"
			elif entry.Ability() == 2:
				msg +=f"HA Only\t"
			else:
				msg += f"Ability {entry.Ability() + 1} Only\t"
			msg += f"G:{entry.Gender()} IV:{entry.FlawlessIVs()}\t"
			msgt = f"{entry.ProbabilitiesAsNumpy()}"
			msg += f"{msgt:18}\t"
			# msg += f"Drop:{entry.DropTableID():X} Bonus:{entry.BonusTableID():X}\t"
			print(msg)
			rank = entry.FlawlessIVs() - 1
			dropid = entry.DropTableID()
			for jj in range(drop.TablesLength()):
				ldt = drop.Tables(jj)
				if dropid == ldt.TableID():
					msg = 'Drop: '
					for kk in range(ldt.EntriesLength()):
						ldte = ldt.Entries(kk)
						if ldte.Values(rank) > 0:
							msg += pmtext.items[ldte.ItemID()] + f'({ldte.Values(rank)}%)' + '  \t'
					print(msg)

			bonusid = entry.BonusTableID()
			for jj in range(bonus.TablesLength()):
				lbt = bonus.Tables(jj)
				if bonusid == lbt.TableID():
					msg = 'Bonus: '
					for kk in range(lbt.EntriesLength()):
						lbte = lbt.Entries(kk)
						if lbte.Values(rank) > 0:
							msg += f'{lbte.Values(rank)} x ' + pmtext.items[lbte.ItemID()] + '\t\t'
					print(msg)
			print('')


import pandas as pd
from datetime import datetime

shape = 1000000
time_lst = []

col_1_and_2 = [i for i in range(shape)]
col_3 = ["a" for i in range(shape)]
col_4 = ["b" for i in range(shape)]
col_5 = ["c" for i in range(shape)]

df_dict = {
    "col_1": col_1_and_2,
    "col_2": col_1_and_2,
    "col_3": col_3,
    "col_4": col_4,
    "col_5": col_5
}

df = pd.DataFrame(df_dict)

## WRITE TO FILES
# HDF
start_time = datetime.now()
df.to_hdf("homework/week_5/data.h5", key="df")
end_time = datetime.now()
time_lst.append(end_time - start_time)

# FEATHER
start_time = datetime.now()
df.to_feather("homework/week_5/data.feather")
end_time = datetime.now()
time_lst.append(end_time - start_time)

# PARQUET
start_time = datetime.now()
df.to_parquet("homework/week_5/data.gzip")
end_time = datetime.now()
time_lst.append(end_time - start_time)

## WRITE TIME TO TXT
with open("homework/week_5/time_used.txt", "w") as f:
    f.write("HDF: " + str(time_lst[0]) + "\n")
    f.write("Feather: " + str(time_lst[1]) + "\n")
    f.write("Parquet: " + str(time_lst[2]) + "\n")


from django.shortcuts import render
from django.http import HttpResponse
# Create your views here.

def home(request):
    return render(request, './first_app/home.html', {'name': 'Tanver Rana Sobur', 'age': 19, 'marks': 90, 'courses': [
        {
            'id': 1,
            'course': 'C',
            'teacher': 'Arifin'
        },
        {
            'id': 2,
            'course': 'C++',
            'teacher': 'Rokey'
        },
        {
            'id': 3,
            'course': 'Database',
            'teacher': 'Iftekhar'
        },
    ], 'name': 'Tanver', 'lst': [1, 2, 4, 7], 'blog': 'I am Tanver Rana Sobur. I am the student of Bangladesh University department of CSE. THis is our blog website.'})

def about(request):
    return render(request, './first_app/about.html', {'author': 'tanver Rana'})


import tensorflow as tf

import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

from model.person_identification_model import PersonIdentificationModel

import numpy as np
import matplotlib.pyplot as plt

class personIden():
    def __init__(self, input_shape):
        self.person_iden = PersonIdentificationModel(input_shape)

        self.person_iden.build(input_shape=input_shape)
        self.person_iden.compile(optimizer=tf.keras.optimizers.Adam(),
                                loss=tf.keras.losses.BinaryCrossentropy(),
                                metrics=['accuracy'])

    def run(self, dataA, dataB, dataFalse, epochs, batch_size):
        dataA = tf.convert_to_tensor(dataA, dtype=np.float32)
        dataB = tf.convert_to_tensor(dataB, dtype=np.float32)
        dataFalse = tf.convert_to_tensor(dataFalse, dtype=np.float32)

        for i in range(epochs):
            F_idx = np.random.randint(0, dataFalse.shape[0], batch_size)
            A_idx = np.random.randint(0, dataA.shape[0], batch_size)

            true_loss = self.person_iden.train_on_batch(dataA.numpy()[A_idx], np.ones((batch_size, 1)))
            false_loss = self.person_iden.train_on_batch(dataFalse.numpy()[F_idx], np.zeros((batch_size, 1)))
            print("{}: true_loss {}, false_loss {}".format(i, true_loss, false_loss))

        r_list = self.person_iden.predict(dataB)
        r_avg = np.sum(r_list) / len(r_list)
        print("result: {:.2f} %".format(r_avg*100))


import sys

sys.path.append('.')
from models import merge_profiles

if len(sys.argv) < 2:
    print('usage: merge_profiles.py NICKNAME [NICKNAMES_TO_MERGE...]', file=sys.stderr)
    sys.exit(1)

nickname = sys.argv[1]
other_nicknames = sys.argv[2:]

for other_nickname in other_nicknames:
    merge_profiles(nickname, other_nickname)


"""microdatasets package installation metadata."""
from importlib.machinery import SourceFileLoader
from pathlib import Path

from setuptools import find_packages, setup

version = SourceFileLoader(
    "microdatasets.version",
    str(Path(__file__).parent / "src" / "microdatasets" / "version.py"),
).load_module()

with open(Path(__file__).with_name("README.md"), encoding="utf-8") as f:
    long_description = f.read()

setup(
    name="microdatasets",
    description="Simple datasets for evaluating DL algorithms.",
    long_description=long_description,
    long_description_content_type="text/markdown",
    packages=find_packages("src"),
    package_dir={"": "src"},
    version=version.__version__,
    license="MIT",
    author="Guillem Duran Ballester",
    author_email="guillem@fragile.tech",
    url="https://github.com/FragileTech/microdatasets",
    keywords=["Machine learning", "artificial intelligence"],
    test_suite="tests",
    tests_require=["pytest>=5.3.5", "hypothesis>=5.6.0"],
    extras_require={},
    install_requires=["matplotlib", "numpy", "scipy"],
    package_data={
        "": ["README.md"],
        "microdatasets": ["assets/**/*", "assets/**/.*", "tests/**/*", "tests/**/.*"],
    },
    include_package_data=True,
    classifiers=[
        "Development Status :: 3 - Alpha",
        "Intended Audience :: Developers",
        "License :: OSI Approved :: MIT License",
        "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: 3.8",
        "Programming Language :: Python :: 3.9",
        "Programming Language :: Python :: 3.10",
        "Topic :: Software Development :: Libraries",
    ],
)


# -*- coding: utf-8 -*-
import os
import sys
import random
import numpy as np
import json
import time
import copy
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import torchvision.transforms as T
import argparse
from torch_geometric.data import DataLoader
from tensorboardX import SummaryWriter
from tqdm import tqdm
from utils import Logger, set_random_seed, worker_init_fn, compute_grad_norm, GradualWarmupScheduler, wrap_dataloader, \
    load_model, load_model_info
from dataset import GamePatch

# if gpu is to be used
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

parser = argparse.ArgumentParser()
parser.add_argument('--data_path', type=str, required=True,
                    help='input the dataset name')
parser.add_argument('--model_path', type=str)
parser.add_argument('--model', type=str, default="pointconv",
                    help='model name')
parser.add_argument('--aggr', type=str, default='max',
                    help='aggregation function')
args = parser.parse_args()
BATCH_SIZE = 128
CROSS_ENTROPY = True
LOSS_BALANCE = True

data_path = args.data_path
with open(data_path, 'r') as f:
    data = json.load(f)

dataset = GamePatch(data["data"])
model_info = load_model_info(dataset, aggr=args.aggr)
policy_net = load_model(model=args.model, info=model_info)
policy_net.load_state_dict(torch.load(args.model_path)['policy_net'])
policy_net = policy_net.to(device)

train_dataset=dataset
train_dataloader = DataLoader(train_dataset,
                              batch_size=BATCH_SIZE,
                              shuffle=False,
                              drop_last=False,
                              num_workers=8,
                              worker_init_fn=worker_init_fn,
                              )

print("==> data size: {}\n".format(len(dataset)))

policy_net.eval()
true_sample, all_sample = 0, 0
eval_result = []
for i, data_batch in tqdm(enumerate(train_dataloader)):
    # copy data from cpu to gpu
    data_batch = data_batch.to('cuda')

    # forward
    with torch.no_grad():
        outputs = policy_net(data_batch)
        loss = 0
        q = outputs['q']

        actions = q.cpu().argmax(-1).numpy()
        gt_actions = data_batch.y.cpu().argmax(-1).numpy()
        true_flag = actions == gt_actions
        true_sample+=np.sum(true_flag)
        all_sample += actions.shape[0]
        #print(true_flag)
        eval_result.append(true_flag)
print('acc: ',true_sample/all_sample)
eval_result = np.concatenate(eval_result)
np.save(os.path.join(os.path.dirname(args.model_path), "eval_result.npy"), eval_result)


# -*- coding = utf-8 -*-

"""
    author:fengxi
    time:2023/2/17

    project name:pnlp2023
    file name:main.py
    function:
        chapter_1主方法：
            数据加载、预处理
            文本（短评）数据可视化
            数据分析与可视化
"""

import jieba
import pandas as pd
from preprocessing import load_stopwords
from visualization_comments import generate_wordcloud, get_cooccurrence_graph, generate_celebrity_graph
from visualization_data import analysis_location

# 加载自定义词典
jieba.load_userdict("./dict/custom_dictionary.txt")

def load_pre_data(filename):
    """
        加载全部数据
        对评论进行预处理
    :param filename:
    :return:
    """
    # === 加载全部数据
    df = pd.read_csv(filename, encoding="utf-8")

    # 提取每一列数据
    user_name = df["用户名"].values.tolist()
    video_status = df["电视剧观看状态"].values.tolist()
    comment_score = df["评论分数"].values.tolist()
    comment_level = df["评论等级"].values.tolist()
    comment_time = df["评论时间"].values.tolist()
    comment_location = df["评论位置"].values.tolist()
    comment_vote_count = df["评论点赞数"].values.tolist()
    comment_content = df["评论"].values.tolist()

    # === 对评论进行预处理
    # 加载停用词词典
    stopwords = load_stopwords()

    comment_seg_list = []  # 分词后的评论
    for comment in comment_content:
        # 简单预处理
        comment = comment.replace("\r", "")
        # 分词
        comment_seg = []
        for words in jieba.lcut(comment):
            # 去停用词（包括中文标点符号）
            if words in stopwords:
                continue
            # 过滤单个字符的词
            if len(words) < 2:
                continue
            # 过滤数字
            if words.isdigit():
                continue
            comment_seg.append(words)
        comment_seg_list.append(comment_seg)

    return user_name, video_status, comment_score, comment_level, comment_time, \
        comment_location, comment_vote_count, comment_seg_list

def main():
    """
        数据分析主流程：
            1、短评分析与可视化：词云图、词共现关系图、人物关系图谱
            2、数据分析与可视化
    :return:
    """
    # 加载和预处理数据
    user_name, video_status, comment_score, comment_level, comment_time, comment_location, \
    comment_vote_count, comment_content = load_pre_data("./data/狂飙.csv")

    # 生成词云图
    generate_wordcloud(comment_content)

    # 获得词共现关系图
    get_cooccurrence_graph(comment_content)

    # 构建人物关系图谱
    generate_celebrity_graph()

    # 统计分析评论位置分布情况
    analysis_location(comment_location)

if __name__ == '__main__':
    # 主方法
    main()
    print("数据分析完成...")

    pass


from __future__ import absolute_import, division, print_function
import pathlib
import random
import tensorflow as tf
from tensorflow import keras
from sklearn.preprocessing import LabelBinarizer
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report

import cv2

import numpy as np
#import matplotlib.pyplot as plt

tf.enable_eager_execution()

import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

AUTOTUNE = tf.data.experimental.AUTOTUNE

#image and data pre-processing

#data directory for images
data_dir = "./ISIC_images"
#root paths for data
data_root = pathlib.Path(data_dir)

#pathways to images
for item in data_root.iterdir():
  print(item)

#pathway to each individual image
all_image_paths = list(data_root.glob('*/*'))
all_image_paths = [str(path) for path in all_image_paths]
random.shuffle(all_image_paths)

#verifier that pathways are correct
for image in all_image_paths:
  print(image)

#label_names created/generated from filenames
label_names = sorted(item.name for item in data_root.glob('*/') if item.is_dir())
print(label_names)

#unnecessary - corresponds each label to a give index
label_to_index = dict((name, index) for index, name in enumerate(label_names))
print(label_to_index)

#assigns given medical condition to each image
all_image_labels = list(pathlib.Path(path).parent.name for path in all_image_paths)
print("First 10 labels indices: ", all_image_labels[:10])

labels = []

#assigns an array of the corresponding labels
for label in all_image_labels:
  labels.append(label)

img_data = []
print(len(all_image_paths))
for image_path in all_image_paths:
  image = cv2.imread(image_path)
  #print(image)
  #image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
  image = cv2.resize(image, (32, 32)).flatten()
  print(image.shape)
  img_data.append(image)

img_data = np.array(img_data, dtype="float")/255.0
labels = np.array(labels)

(trainX, testX, trainY, testY) = train_test_split(img_data, labels, test_size=0.25, random_state=42)
lb = LabelBinarizer()
trainY = lb.fit_transform(trainY)
testY=lb.transform(testY)

train = []
for p in trainY:
  train.append(np.argmax(p))
print(train)

train = np.array(train)

model = keras.Sequential([
  keras.layers.Dense(1024, input_shape=(3072,), activation="relu"),
  keras.layers.Dense(4, activation="softmax" ),

])

sgd = keras.optimizers.SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)

model.compile(optimizer='sgd', loss = 'sparse_categorical_crossentropy', metrics=['accuracy'])

model.fit(trainX, train, epochs=5, batch_size=32)

predictions = model.predict(testX, batch_size=32)

train = []
for p in testY:
  train.append(np.argmax(p))

train = np.array(train)

#print(classification_report(train.argmax(axis=1), predictions.argmax(axis=1), target_names=lb.classes_))

test_loss, test_acc = model.evaluate(testX, train)
print(test_acc)

for p in predictions:
  print(np.argmax(p))
print(train)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Apr 12 10:52:20 2020

@author: aadityabhatia

Info: This module builds a hierarchical structure for brands and products where
Brands/ foler has multiple brands and
each brand has multiple product folders.
Brand folders have brand csvs and product folders have product csvs.
"""

import pandas as pd
import os
import spacy
import unidecode
from nltk.corpus import stopwords
import string
import argparse

nlp = spacy.load("en_core_web_sm", disable=['parser', 'tagger', 'ner'])
stops = stopwords.words("english")
punctuations = string.punctuation
for p in punctuations:
    stops.append(p)
stops.append('')

# arg parser
parser = argparse.ArgumentParser(description='Builds the models to test accuracy')
parser.add_argument('-d', '--dirName', default=None)
parser.add_argument('-l', '--lab_d', default='reference_data.csv')

args = parser.parse_args()
dirName = args.dirName
if not dirName:
    current_directory = os.getcwd()
    current_directory = current_directory.replace('\\', '/')
    dirName = f'{current_directory}/Data/'

lab_d = args.lab_d

def getLemma(comment):
    lemmatized = []
    tokens = nlp(unidecode.unidecode(str(comment).lower()))
    for token in tokens:
        token = token.lemma_
        if token not in stops:
            lemmatized.append(token)
    return ' '.join(lemmatized)

if __name__ == "__main__":
    lab_df = pd.read_csv(lab_d)

    # Preprocessing the labels file for future use:
    lab_df['query_lemma'] = lab_df['query'].apply(getLemma)

    if not os.path.exists(dirName):
        os.mkdir(dirName)

    # making brand path
    if not os.path.exists(dirName + 'Brands'):
        os.mkdir(dirName + 'Brands')
    if not os.path.exists(dirName + 'SavedModels'):
        os.mkdir(dirName + 'SavedModels')

    # lists for unprocessed brands (can save for if needed)
    unprocessed_brands = []
    unprocessed_products = []

    # making file structures in local dir
    for brand, brand_grouper in lab_df.groupby('brand'):
        brand = brand.replace('/', '-')
        brand_path = dirName + 'Brands/' + brand + '/'

        # making brand files
        if len(brand_grouper) > 5:
            if not os.path.exists(brand_path):
                os.mkdir(brand_path)
            brand_grouper.to_csv(brand_path + brand + '.csv')

            # making product files
            for product, product_grouper in brand_grouper.groupby('product_id'):
                product_path = brand_path + product
                if len(product_grouper) > 5:
                    if not os.path.exists(product_path):
                        os.mkdir(product_path)
                    product_grouper.to_csv(product_path + '/' + product + '.csv')
                else:
                    unprocessed_products.append(product)
        else:
            unprocessed_brands.append(brand)

    print('\n\n\n...Estimations')
    print(len(unprocessed_brands) / lab_df.brand_id.nunique() * 100, '% brands not processed')
    print(len(unprocessed_products) / lab_df.product_id.nunique() * 100, '% products not processed')


#!/usr/bin/python3
# https://practice.geeksforgeeks.org/problems/number-to-words/0

tensToWords = {1000: "Thousand", 100: "Hundred"}
tens = {
    1: "Ten", 2: "Twenty", 3: "Thirty", 4: "Forty", 5: "Fifty",
    6: "Sixty", 7: "Seventy", 8: "Eighty", 9: "Ninety"}
ones = {
    1: "One", 2: "Two", 3: "Three", 4: "Four", 5: "Five", 6: "Six",
    7: "Seven", 8: "Eight", 9: "Nine", 0: "Zero"}
tensP = {
    1: "Eleven" , 2: "Twelve", 3: "Thirteen", 4: "Fourteen",
    5: "Fifteen", 6: "Sixteen" , 7: "Seventeen" , 8: "Eighteen" , 9: "Nineteen"
}

## check the spellings of 19 and 90

def sol(n):
    l = len(str(n))-1
    t = 10**l
    res = ""

    if n == 0:
        return ones[0]
    # Special case

    while n > 0:
        subRes = ""
        pv = n//t

        if t == 100 and n%t:
        # There is a connector and after hundred but only if something
        # follows
            conn = " and "
        else:
            conn = " "

        if t == 1:
            subRes = ones[pv]
        elif t == 10:
            if pv == 1 and n%t > 0:
            # To handle cases from 11-19
                subRes = tensP[n%t]
                n = 0
            # Set n as 0 because we do not want processing for ones from here on
            elif pv >= 1:
                subRes = tens[pv] + conn
        else:
            if pv >= 1:
                subRes = ones[pv] + " " + tensToWords[t] + conn
            # pv >= 1 is there to handle cases like 505, subRes goes empty

        res = res + subRes
        n = n%t
        t = t//10

    return res.strip()

nums = [10, 1000, 505, 36, 19, 2105, 1008, 100, 9999, 3037, 0]
for num in nums:
    print(sol(num))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# project = https://github.com/lowliness9/pytools.git
# author = lowliness9
# date: 2021-04-23
from concurrent.futures import ThreadPoolExecutor
import threading
import requests
import sys
import time
import os

requests.packages.urllib3.disable_warnings()

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36',
    'Connection':'close'
}

workDIR = sys.argv[1]

lock = threading.Lock()

todayFlag = time.strftime("%Y-%m-%d-", time.localtime())

aliveProxyList = []

def outputAliveProxy(proxy):
    lock.acquire()
    print(proxy)
    with open(os.path.join(workDIR,todayFlag + 'proxy.txt'),'a+') as f:
        if proxy not in aliveProxyList:
            f.write(proxy.strip()+'\n')
    lock.release()

def ProxyTest(proxyList):
    procotol = proxyList[0]
    IP = proxyList[1]
    PORT = proxyList[2]

    proxies = {"http": "{}://{}:{}".format(procotol,IP,PORT), "https": "{}://{}:{}".format(procotol,IP,PORT)}
    try:
        request = requests.get(url = 'http://icanhazip.com/', proxies = proxies,timeout = (5,3) , verify = False,headers = headers)
        if request.text.strip() == IP:
            lock.acquire()
            print(' '.join(proxyList))
            lock.release()
            aliveProxyList.append(' '.join(proxyList))
    except:
        pass

def todayProxyTest(proxyList):
    procotol = proxyList[0]
    IP = proxyList[1]
    PORT = proxyList[2]

    proxies = {"http": "{}://{}:{}".format(procotol,IP,PORT), "https": "{}://{}:{}".format(procotol,IP,PORT)}
    try:
        request = requests.get(url = 'http://icanhazip.com/', proxies = proxies,timeout = (5,3) , verify = False,headers = headers)
        if request.text.strip() == IP:
            outputAliveProxy(' '.join(proxyList))
    except:
        pass

def Builder():
    proxyList = []
    proxyfileList = open(os.path.join(workDIR,sys.argv[2]),'r+').readlines()
    for proxy in proxyfileList:
        sinProxy = proxy.split()
        if sinProxy:proxyList.append(sinProxy)
    return proxyList

def checkHistory():
    if not os.path.exists(os.path.join(workDIR,todayFlag + 'proxy.txt')):
        with open(os.path.join(workDIR,todayFlag + 'proxy.txt'),'a+') as f:
            for proxy in aliveProxyList:
                if proxy:
                    f.write(proxy+'\n')
        os.remove(os.path.join(workDIR, sys.argv[2]))
    else:
        todayProxyList = []
        for proxy in open(os.path.join(workDIR,todayFlag + 'proxy.txt'),'r+').readlines():
            proxyT = proxy.strip().split()
            if proxyT:todayProxyList.append(proxyT)
        os.remove(os.path.join(workDIR,todayFlag + 'proxy.txt'))
        with ThreadPoolExecutor(20) as executor:
            executor.map(todayProxyTest,todayProxyList)
        for proxy in aliveProxyList:
            with open(os.path.join(workDIR, todayFlag + 'proxy.txt'), 'a+') as f:
                f.write(proxy.strip() + '\n')
        os.remove(os.path.join(workDIR,sys.argv[2]))

if __name__ == '__main__':

    proxyList = Builder()
    with ThreadPoolExecutor(20) as executor:
        executor.map(ProxyTest,proxyList)
    checkHistory()


from ortools.linear_solver import pywraplp
from Components.Food import FoodItem, ListFoodItem, ConstraintList, Constraint
from Components.utils import update_json
import pandas as pd

import streamlit as st
import json

#Global variable
food_list = ListFoodItem([])
food_list.initialize_me()

constraint_list = ConstraintList([])
constraint_list.initialize_me()

def MinimizeFunction(dataframe, constraints):
    solver = pywraplp.Solver.CreateSolver('SCIP')

    foods = [solver.NumVar(0.0, solver.infinity(), name=name) for name in dataframe['name']]

    nvariables = solver.NumVariables()

    objective = solver.Objective()

    for constraint in constraints.constraintslist:
        solver.Add(solver.Sum([foods[i]*dataframe[constraint.name][i] for i in range(len(foods))]) >= constraint.value)

    for i ,food in enumerate(foods):
        objective.SetCoefficient(food, dataframe['cost'][i])

    solver.Add(solver.Sum([food for food in foods]) == 1)
    objective.SetMinimization()
    status = solver.Solve()

    if status == pywraplp.Solver.OPTIMAL:
        st.markdown('---------------------------------------')
        st.write('Solution:')
        st.write('Objective value =', solver.Objective().Value())
        for food in foods:
            if food.solution_value() > 0.01:
                st.write(food.name() , food.solution_value(),"kg")
    else:
        st.write('The problem does not have an optimal solution.')

    return {
        'nvariables':nvariables,
        'status':status
    }

# Instantiate a Glop solver and naming it.
def AddFoodForms(food_list):
    with st.expander("Add Food Forms 🥐"):
        #Initizalizing Forms
        form = st.form(key="add_form")

        #Food Inputs
        name = form.text_input("Name of the food")
        cost = form.number_input("Cost ($/kg)")
        protein = form.number_input("Protein")
        calcium = form.number_input("Calcium")

        #Submit button
        submit = form.form_submit_button('Submit')

        #onSubmit
        if submit:
            #Create and append the food to the foodlist
            newfood = FoodItem(name=name, cost=cost, protein= protein, calcium = calcium)
            food_list.foodlist.append(newfood)

            #Write item into the json file
            update_json(item = food_list,filename ='foods.json')

            return newfood
        else:
            return

def FoodFormsConstraints(constraint_list):
    with st.expander("Add the constraints ⛓"):
        form_2 = st.form(key="constraint_forms")

        calcium_constraint =  form_2.number_input("Maximum Calcium", key="calcium_constraint")
        protein_constraint = form_2.number_input("Maximum Protein", key="protein_constraint")

        submit_button = form_2.form_submit_button('Submit')

        if submit_button:
            constraint01 = Constraint(name="calcium", value=calcium_constraint)
            constraint02 = Constraint(name="protein", value=protein_constraint)

            constraint_list.update_constraint_list(constraint01)
            constraint_list.update_constraint_list(constraint02)

            update_json(item = constraint_list, filename ='constraints.json')
    return

def FoodMixesPages():

    FoodFormsConstraints(constraint_list)
    AddFoodForms(food_list)

    df = food_list.food_list_to_pandas()
    constraint_df = constraint_list.data_to_pandas()

    if not df.empty:
        st.markdown("## Dataframe with the food")
        st.dataframe(df)
        cleardataframe = st.button(" 🗑 Delete Dataframe")
        if cleardataframe:
            food_list.foodlist.clear()
            update_json(item=food_list, filename='foods.json')

        res = MinimizeFunction(dataframe=df, constraints=constraint_list)
        status = res['status']
        st.write(res["nvariables"])
    else:
        st.markdown("No item added to the dataframe 😓")

    if not constraint_df.empty:
        st.dataframe(constraint_df)
        clearconstraints = st.button(" 🗑 Delete Constraint")
        if clearconstraints:
            constraint_list.constraintslist.clear()
            update_json(item=constraint_list, filename='constraints.json')
    else:
        st.markdown("No constraint added")
    return


from actions.utils import create_action
from bookmarks.common.decorators import ajax_required
from bookmarks.common.utils import is_ajax
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.core.exceptions import ObjectDoesNotExist
from django.core.paginator import (
    Paginator,
    EmptyPage,
    PageNotAnInteger,
)
from django.http import JsonResponse
from django.shortcuts import (
    render,
    redirect,
    get_object_or_404,
    HttpResponseRedirect,
    HttpResponse,
)
from django.views.decorators.http import require_POST

from .forms import ImageCreateForm
from .models import Image

@login_required
def image_list(request):
    images = Image.objects.all()
    paginator = Paginator(images, per_page=8)
    page = request.GET.get('page')
    ajax = is_ajax(request)
    try:
        images = paginator.page(page)
    except PageNotAnInteger:
        images = paginator.page(number=1)
    except EmptyPage:
        if ajax:
            return HttpResponse('')
        images = paginator.page(number=paginator.num_pages)
    if ajax:
        return render(request, 'images/image/list_ajax.html',
                      {'section': 'images', 'images': images})
    return render(request, 'images/image/list.html',
                  {'section': 'images', 'images': images})

@login_required
@require_POST
@ajax_required
def image_like(request):
    image_id = request.POST.get('id')
    action = request.POST.get('action')
    if image_id and action:
        try:
            image = Image.objects.get(id=image_id)
            if action == 'like':
                image.users_like.add(request.user)
                create_action(request.user, 'liked', image)
            else:
                image.users_like.remove(request.user)
            return JsonResponse({'status': 'ok'})
        except ObjectDoesNotExist:
            messages.error(request, 'An error occurred in adding like process')
            return HttpResponseRedirect('/account/dashboard/')

    return JsonResponse({'status': 'ok'})

def image_detail(request, id, slug):
    image = get_object_or_404(Image, id=id, slug=slug)
    return render(request, 'images/image/detail.html',
                  {'section': 'images', 'image': image})

@login_required
def image_create(request):
    if request.method == 'POST':
        image_form = ImageCreateForm(data=request.POST)
        if image_form.is_valid():
            new_image = image_form.save(commit=False)
            new_image.user = request.user
            new_image.save()
            create_action(request.user, 'added ', new_image)
            messages.success(request, 'Image has been added.')
            return redirect(new_image.get_absolute_url())
        else:
            messages.error(request, 'An error occurred while image was adding')
    else:
        image_form = ImageCreateForm(data=request.GET)
    return render(request, 'images/image/create.html',
                  {'section': 'gallery', 'image_form': image_form})