# winequality-white.csv
# RandomForest로 모델 만들기
# 분류모델인듯

import numpy as np
import pandas as pd
from sklearn.datasets import load_wine
from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler

from sklearn.svm import LinearSVC, SVC
from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor
from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
from sklearn.metrics import accuracy_score

# 1. data
dataset = pd.read_csv("./data/csv/winequality-white.csv",header=0,index_col=None,sep=";")

print(dataset)
print(dataset.shape)

datasets = pd.DataFrame(dataset)
datasets = datasets.values
print(dataset.shape)
print(type(datasets))
x = datasets[:,:11]
y = datasets[:,11:]
print(x.shape)
print(y.shape)
print(x)
print(y)

from sklearn.model_selection import train_test_split
x_train, x_test , y_train  , y_test = train_test_split(x, y, train_size=0.8, random_state=1,shuffle=True)

scaler = StandardScaler()
scaler.fit(x_train)
x_train = scaler.transform(x_train)
x_test = scaler.transform(x_test)
# x_train = scaler.transform(x_train).astype("float")
# x_test = scaler.transform(x_test).astype("float")
# y_train = y_train.astype("float")
# y_test = y_test.astype("float")

# 2. 모델
# model = LinearSVC()
# model = SVC()
# model = KNeighborsClassifier()
# model = RandomForestClassifier()
# model = KNeighborsRegressor()
model = RandomForestRegressor()

# 3. 훈련
model.fit(x_train,y_train)
x_pred = x_test[:10]
y_real = y_test[:10]

# 4. 평가 예측
y_predict = model.predict(x_pred)
print("y_real :  ",y_real)
print("predict : ",y_predict)

model_score = model.score(x_test,y_test)
print("model_score : ",model_score)

acc_score = accuracy_score(y_real,y_predict.round())
print("acc_score :   ",acc_score)
# 분류 => accuracy_score
# 회귀 => r2_score

# R2
# from sklearn.metrics import r2_score
# print("R2 : ",r2_score(y_real,y_predict))

# ==== 분류

# LinearSVC
# model_score :  0.5265306122448979
# acc_score :    0.2

# SVC
# model_score :  0.5561224489795918
# acc_score :    0.3

# KNeighborsClassifier
# model_score :  0.5571428571428572
# acc_score :    0.2

# RandomForestClassifier
# model_score :  0.6836734693877551
# acc_score :    0.4

# ==== 회귀

# KNeighborsRegressor
# model_score :  0.35631420985015383
# acc_score :    0.2

# RandomForestRegressor
# model_score :  0.5219016183504787
# acc_score :    0.4


from _data_preprocess2 import DataPrep

from sklearn.model_selection import train_test_split
from sklearn import metrics
import datetime
import lightgbm as lgb

class Model(object):

    def __init__(self, db=None):
        self.db = db
        self.data_pre = DataPrep(db=self.db)

    def get_label(self, data):
        data['post1'] = data.sort_values('trade_date').groupby('ts_code')['daily-close-0'].shift(1)
        data['post3'] = data.sort_values('trade_date').groupby('ts_code')['daily-close-0'].shift(3)
        data['post5'] = data.sort_values('trade_date').groupby('ts_code')['daily-close-0'].shift(5)
        data['y1'] = data['post1'] / data['daily-close-0']
        data['y2'] = data['post3'] / data['daily-close-0']
        data['y3'] = data['post5'] / data['daily-close-0']

        cp_col = ['ts_code']
        data_col = ['trade_date']
        y_cols = ['post1', 'post3', 'post5', 'y1', 'y2', 'y3']
        ft_cols = list(set(data.columns.tolist()) - set(cp_col + data_col + y_cols))

        return data

    def get_train_data(self, start_date=None, end_date=None):
        if (start_date is None) or (end_date is None):
            tmp = datetime.datetime.now()
            end_date = tmp.strftime('%Y%m%d')
            start_date = (tmp-datetime.timedelta(days=30)).strftime('%Y%m%d')
        data = self.data_pre.get_data(start_date=start_date, end_date=end_date)
        data = self.get_label(data)

        cp_col = ['ts_code']
        data_col = ['trade_date']
        y_cols = ['post1', 'post3', 'post5', 'y1', 'y2', 'y3']
        ft_cols = list(set(data.columns.tolist()) - set(cp_col + data_col + y_cols))

        tmp = data[~data['y1'].isna()]
        X = tmp[ft_cols].to_numpy()
        Y = (tmp['y1'] > 1.05).astype('int').to_numpy()

        return X, Y

    def get_predict_data(self):
        date = datetime.date().strftime('%Y%m%d')
        data = self.data_pre.get_data(date=date)

        cp_col = ['ts_code']
        data_col = ['trade_date']
        y_cols = ['post1', 'post3', 'post5', 'y1', 'y2', 'y3']
        ft_cols = list(set(data.columns.tolist()) - set(cp_col + data_col + y_cols))

        X = data[ft_cols].to_numpy()

        return X

    def train(self):
        self.X, self.Y = self.get_train_data()
        x_train, x_test, y_train, y_test = train_test_split(self.X, self.Y, test_size=0.25)
        gbm = lgb.LGBMClassifier()

        gbm.fit(x_train, y_train)

        y_pre = gbm.predict(x_test, num_iteration=gbm.best_iteration_)

        self.acc = metrics.accuracy_score(y_test, y_pre)
        self.precision = metrics.precision_score(y_test, y_pre)
        self.recall = metrics.recall_score(y_test, y_pre)
        self.f1_score = metrics.f1_score(y_test, y_pre)
        self.auc = metrics.roc_auc_score(y_test, y_pre)
        self.total_num = y_test.shape[0]
        self.p_num = y_test.reshape(-1, ).sum()
        self.n_num = self.total_num - self.p_num
        self.pp_num = y_pre.reshape(-1, ).sum()
        self.pn_num = self.total_num - self.pp_num
        self.cf_mt = metrics.confusion_matrix(y_test, y_pre)

        self.model = gbm
        return gbm

    def predict(self, data=None):
        x = self.get_predict_data(data)

        y_pre = self.model.predict(x)
        return y_pre


import pymongo
import json

file_path = '/home/estudiante/project/web-server/tweets/@VickyDavilaH.json'

def loadFile():
	with open(file_path) as data_file:
		file_str = data_file.read()
		file_str = '[' + file_str + ']'
		file_str = file_str.replace('}{','},{')
		return json.loads(file_str)

client = pymongo.MongoClient("mongodb://clusterbigdata.placeholder.edu")
db = client.Grupo04

tweets = loadFile()
for batch in tweets:
	for status in batch['statuses']:
		db.tweets.insert_one(status)

Assignment 3 - Twitter sentiment analysis using MongoDB/web-server/tweetLoader.py


"""
The power of the string is the maximum length of a non-empty substring that contains only one unique character.
Given a string s, return the power of s.

Input: s = "abbcccddddeeeeedcba"
Output: 5
Explanation: The substring "eeeee" is of length 5 with the character 'e' only.
"""

class Solution:
    def maxPower(self, s: str) -> int:
        N = len(s)
        cur = 1
        max_length = 1
        for i in range(1,N):
            if (s[i] == s[i-1]):
                cur += 1
            else:
                max_length = max(max_length,cur)
                cur = 1
        max_length = max(max_length,cur)
        return max_length
    def maxPower1(self, s: str) -> int:
        left = 0
        right = 0
        N = len(s)
        max_length = 0
        while (left <= right and right < N):
            while (right < N and s[left] == s[right]):

                right += 1
            max_length = max(max_length, right-1-left+1)
            left = right
        return max_length

s = "abbcccddddeeeeedcba"
max_length = Solution().maxPower(s)
print(max_length)


handle = open("data/m_cold.fasta", "r")
handle.readline()

from Bio import SeqIO
for record in SeqIO.parse("data/m_cold.fasta", "fasta"):
    print(record.id, len(record))

from Bio import SeqIO
handle = open("data/m_cold.fasta", "r")
for record in SeqIO.parse(handle, "fasta"):
    print(record.id, len(record))
handle.close()

my_info = 'A string\n with multiple lines.'
print(my_info)

from io import StringIO
my_info_handle = StringIO(my_info)
first_line = my_info_handle.readline()
print(first_line)

second_line = my_info_handle.readline()
print(second_line)

python/23 - Appendix, Useful stuff about Python.py


from unittest.mock import patch

import pytest
from conftest import (
    BUCKET_NAME,
    LOGFILE_NO_GGCANARY,
    LOGFILE_W_GGCANARY,
    make_trigger_event,
)

from lambda_py import lambda_function
from lambda_py.notifiers.ses_notifier import SESNotifier
from lambda_py.notifiers.webhook_notifier import IWebhookNotifier

@pytest.mark.parametrize(
    "key,nb_records",
    (
        (LOGFILE_W_GGCANARY, 3),
        (LOGFILE_NO_GGCANARY, 3),
    ),
)
def test_fetch_logs(key, nb_records):
    records = lambda_function.fetch_log_records(bucket=BUCKET_NAME, key=key)
    assert len(records) == nb_records

def test_get_report_entries_ggcanary():
    records = lambda_function.fetch_log_records(
        bucket=BUCKET_NAME, key=LOGFILE_W_GGCANARY
    )
    report_entries = lambda_function.get_report_entries(records)
    assert len(report_entries) == 2

def test_get_report_entries_no_ggcanary():
    records = lambda_function.fetch_log_records(
        bucket=BUCKET_NAME, key=LOGFILE_NO_GGCANARY
    )
    report_entries = lambda_function.get_report_entries(records)
    assert len(report_entries) == 0

@patch.object(SESNotifier, "send_notification")
@patch.object(IWebhookNotifier, "send_notification")
def test_lambda_handler_called(
    mock_method_1,
    mock_method_2,
    mock_ses_validation,
):

    notifiers = lambda_function.load_notifiers()
    assert len(notifiers) == 3
    assert notifiers[0].__class__ == SESNotifier

    event = make_trigger_event(bucket_name=BUCKET_NAME, key=LOGFILE_W_GGCANARY)
    lambda_function.handler(event)
    mock_method_1.assert_called()
    mock_method_2.assert_called()

@patch.object(SESNotifier, "send_notification")
@patch.object(IWebhookNotifier, "send_notification")
def test_lambda_handler_not_called(
    mock_method_1,
    mock_method_2,
):

    notifiers = lambda_function.load_notifiers()
    assert len(notifiers) == 3
    assert notifiers[0].__class__ == SESNotifier

    event = make_trigger_event(bucket_name=BUCKET_NAME, key=LOGFILE_NO_GGCANARY)
    lambda_function.handler(event)
    mock_method_1.assert_not_called()
    mock_method_2.assert_not_called()


"""
Aidan Duffy
Class: CS 521 - Fall 2
Date: December 8, 2020
Homework Problem 5.6.3
Description: This takes in three numbers and performs some quick calculations.
"""

def main():
    nums = input("Please input 3 numbers separated by commas (ie 1,2,3): ")
    num_array = []
    try:
        num_str_array = nums.split(",")
        for i in range(0,3):
            num_array.append(float(num_str_array[i]))
        if num_array[1] == 0:
            print("You are attempting to divide by zero!")
            raise Exception(ZeroDivisionError)
    except:
        print("Please try again with 3 numbers, separated by commas!")
        raise Exception(ValueError)
    answer = (num_array[0]/num_array[1]) + num_array[2]
    print(num_array[0], "/", num_array[1], "+", num_array[2], "=", '{:.2f}'.format(answer))

if __name__ == "__main__":
    main()


from collections import defaultdict
from math import log

from trainingUtils import *

import numpy as np

# 多项式模型
class MultinomialNB(object):

    def __init__(self, alpha=1.0):
        self.word_count = 0
        self.vocab = []
        self.prior_prob = []
        self.alpha = alpha
        self.classes = None
        self.conditional_prob = None
        self.count = []

    def words2vec(self, vocabList, inputSet):
        return_vec = [0] * len(vocabList)
        for word in inputSet:
            if word in vocabList:
                return_vec[vocabList.index(word)] += 1

        return return_vec

    # 构造向量空间
    def make_vec(self, ham_path='./email/ham', spam_path='./email/spam', train=False):
        words_ham, vocab_ham = load_file(ham_path)
        words_spam, vocab_spam = load_file(spam_path)
        self.count.append(len(words_ham))
        self.count.append(len(words_spam))
        self.word_count = self.count[0] + self.count[1]
        return self.vectorize(words_ham, words_spam, vocab_ham, vocab_spam, train)

    def make_vec_from_list(self, ham_list, spam_list, train=False):
        print("loading ham files...")
        words_ham, vocab_ham = load_file_from_list(ham_list)
        print("loading spam files...")
        words_spam, vocab_spam = load_file_from_list(spam_list)
        self.count.append(len(words_ham))
        self.count.append(len(words_spam))
        self.word_count = self.count[0] + self.count[1]
        return self.vectorize(words_ham, words_spam, vocab_ham, vocab_spam, train)

    def vectorize(self, words_ham, words_spam, vocab_ham, vocab_spam, train):
        tmp = set([])
        if train:
            self.vocab = vocab_ham | vocab_spam
        else:
            tmp = (vocab_ham | vocab_spam) - self.vocab
        vo = list(self.vocab) + list(tmp)
        labels = []
        X = []
        for document in words_ham:
            labels.append(0)
            X.append(self.words2vec(vo, document))

        for document in words_spam:
            labels.append(1)
            X.append(self.words2vec(vo, document))

        X = np.array(X)
        y = np.array(labels)
        return X, y

    def fit(self, X, y):
        self.classes = np.unique(y)
        self.update_prior_prob(y)
        # P( xj | y=ck )
        self.conditional_prob = {}
        for c in self.classes:
            self.conditional_prob[c] = defaultdict(lambda: defaultdict(lambda: log(0.5)))
            for i in range(len(X[0])):
                feature = X[np.equal(y, c)][:, i]
                # print(np.equal(y,c))
                # print(X[np.equal(y, c)])
                self.conditional_prob[c][i] = self.update_feature_prob(feature, c)
        return self

    def update_prior_prob(self, y):
        # P(y=ck)
        class_num = len(self.classes)

        sample_num = self.word_count
        for c in self.classes:
            self.prior_prob.append(log(self.count[c] + self.alpha) - log(sample_num + class_num * self.alpha))

    def update_feature_prob(self, feature, c):
        values = np.unique(feature)
        total = self.count[c]
        prob = defaultdict(lambda: log(self.alpha) - log(total + len(values) * self.alpha))
        for v in values:
            prob[v] = log(np.sum(np.equal(feature, v)) + self.alpha) - log(total + len(values) * self.alpha)
            # print(prob[v])
        return prob

    def predict_sample(self, x):
        label = -1
        max_prob = -999

        # 先验概率*条件概率
        for index in range(len(self.classes)):
            label_tmp = self.classes[index]
            prior = self.prior_prob[index]
            conditional = 0.0
            feature_prob = self.conditional_prob[label_tmp]
            # print(feature_prob)
            for i in range(len(feature_prob)):
                conditional += feature_prob[i][x[i]]
                # if feature_prob[i][x[i]] > 0:
                #     print(feature_prob[i][x[i]])

            # argmax
            if prior + conditional > max_prob:
                max_prob = prior + conditional
                label = label_tmp

        return label

    def predict(self, X):
        labels = []
        for i in range(X.shape[0]):
            label = self.predict_sample(X[i])
            labels.append(label)
        return labels


##store lines function to  import file and store each line into a list
def store_lines():
    ##list creation
    st_list= []
    print('Normal Text:')
    ##open sample tect as variable text
    with open('sample.txt') as text:
        #loop to iterate through sample.txt lines and store them to list
        while True:
            ##store lines in the sample text to variable called lines
            lines = text.readline()
            ##strip white space at begining and end
            lines.strip()
            ##print the original document
            print(lines)
            #insert the lines from sample.txt at st_list at pointer 0 to reverse list
            st_list.insert(0, lines)
            #break case
            if not lines:
                break
    #return st_list to print_story function
    print_story(st_list)

##print story function to return function
def print_story(st_list):
    ##reverse string created with no entry
    reverse = ''
    ##for loop to iterate over st_list to join list to string
    for i in st_list:
                line=str(i)
                ##enter new line for spacing
                reverse += "\n"
                ##compound the line to the reverse string variable to make one string
                reverse += line
    ## print the reversed string
    print("Reversed Text:")
    print(reverse)

##call first function
store_lines()


import os

def memory_usage(path):
  print(f'PathL {path}')

  # check for file/folder exisiting
  if not os.path.exists(path):
    return 0

  total = os.path.getsize(path)

  if os.path.isdir(path):
    for filename in os.listdir(path):
      file_path = os.path.join(path, filename)
      total += memory_usage(file_path)

  return total

print(memory_usage('/Users/poval/practice/algorithms'))


class Solution:
    def permuteUnique(self, nums: List[int]) -> List[List[int]]:
        def v1(nums):
            vis =set()
            if len(nums)<=1:
                return [nums]
            else:
                res = []
                for i in range(len(nums)):
                    if nums[i] not in vis:
                        tli = [ [nums[i]] + el for el in v1(nums[:i]+nums[i+1:])]
                        vis.add(nums[i])
                        res.extend(tli)
                return res
        return v1(sorted(nums))


import pygame

from src.runner import Runner
from src.enumtypes import Direction
from src.gridutil import grid_index

class Player(Runner):
    """This is the class for a player running around on the grid"""

    def __init__(self, cfg, screen):
        """Constructor for the Player class"""
        self.caught = False

        super().__init__(cfg, screen)

    def draw(self):
        """Draws the player"""
        if self.x is not None and self.y is not None:
            x = int(self.x * self.cfg.grid_height + self.cfg.grid_origin[0] + 0.5)
            y = int(self.y * self.cfg.grid_height + self.cfg.grid_origin[1] + 0.5)

            if self.caught:
                color = self.cfg.player_color_caught
            else:
                color = self.cfg.player_color

            pygame.draw.circle(self.screen, color, (x, y), int(.5 * self.cfg.player_size), 0)

    def update_position(self, dt_ms):
        """updates the position of the player"""

        if dt_ms > 0 and not self.caught:
            # reset the node visited (but only if it's not the second part of a partial movement)
            if self.dt_ms_rem == 0:
                self.node_visited_x = None
                self.node_visited_y = None

            node_crossed = False

            can_move = self.can_move_on()

            # backup from the the player started moving
            x_start = self.x
            y_start = self.y

            if can_move:
                node_crossed = self.move(dt_ms)

            if x_start != self.x or y_start != self.y:
                # need to update the corresponding gridline
                self.update_gridline_done((x_start, y_start), (self.x, self.y))

            # if a node was crossed in the movement above then another move has to be executed
            # (but since the remaining time was recorded internally we don't have to worry
            # about what's remaining)
            if node_crossed:
                # there's a movement direction change "queued". Apply it now. The player risks
                # that this will lead to no movement, because it's not possible. But this has
                # to be left to the player!

                old_direction = self.move_direction
                old_vel = self.vel

                if self.next_direction is not None:
                    self.move_direction = self.next_direction
                    self.next_direction = None

                can_move = self.can_move_on()

                if can_move:
                    self.move(0)
                else:
                    # restore the previous (i.e. current) movement and continue with it if possible. If
                    # possible then the commanded direction change has to be applied later. If cannot continue
                    # with the previous (i.e. current) movement then stand still
                    self.vel = old_vel
                    self.next_direction = self.move_direction
                    self.move_direction = old_direction

                    if self.can_move_on():
                        self.move(0)
                    else:
                        self.vel = 0
                        self.move_direction = None
                        self.next_direction = None

            self.dt_ms_rem = 0

    def command_direction(self, new_move_direction):
        """Changes the movement direction of the player (only if allowed!)"""
        self.next_direction = None  # reset the "queued" movement change
        if self.vel == 0:
            # check if we can start moving in a certain direction
            self.vel = self.cfg.speed
            self.move_direction = new_move_direction
            if not self.can_move_on():
                # cannot move in the desired direction: therefore reset the movement
                self.vel = 0
                self.move_direction = None
        elif self.move_direction != new_move_direction:
            # the player is already moving, let's check if the new commanded direction is immediately taking action
            # or if we just store it
            if (self.move_direction == Direction.up and new_move_direction == Direction.down) or \
                    (self.move_direction == Direction.down and new_move_direction == Direction.up) or \
                    (self.move_direction == Direction.right and new_move_direction == Direction.left) or \
                    (self.move_direction == Direction.left and new_move_direction == Direction.right):
                # a reverse of the direction is commanded, this can immediately by applied
                self.move_direction = new_move_direction
            else:
                # command is "queued" for later action at the next node
                self.next_direction = new_move_direction

    def update_gridline_done(self, p1, p2):
        """Updates the gridlines to mark the next piece as done"""
        ix_v = None
        ix_h = None
        if p1[0] != int(p1[0]) or p1[1] != int(p1[1]):
            ix_v, ix_h = grid_index(p1[0], p1[1])
            x_ref = int(p1[0])
            y_ref = int(p1[1])
        elif p2[0] != int(p2[0]) and p2[1] != int(p2[1]):
            ix_v, ix_h = grid_index(p2[0], p2[1])
            x_ref = int(p2[0])
            y_ref = int(p2[1])

        if (ix_v is not None) and (ix_h is not None):
            if self.move_direction == Direction.up or self.move_direction == Direction.down:
                self.grid.gridlines[ix_v][ix_h].update(p1[1] - y_ref, p2[1] - y_ref)
            else:
                self.grid.gridlines[ix_v][ix_h].update(p1[0] - x_ref, p2[0] - x_ref)


import os
import abc

import matplotlib.pyplot as plt
import torch
import torchvision
from torch.utils.tensorboard import SummaryWriter
from tqdm import tqdm

class GANTrainer:

    def __init__(self, generator, discriminator, g_optimizer, d_optimizer,
                 n_latent=100, device='cpu', writer=None, model_path=None, output_dir=None,
                 ):
        self._generator = generator.to(device)
        self._discriminator = discriminator.to(device)
        self._device = device
        self._g_optimizer = g_optimizer
        self._d_optimizer = d_optimizer
        self._writer = writer
        self._model_path = model_path
        self._output_dir = output_dir
        self._n_latent = n_latent

        self._global_step = 0
        self._z_fixed = torch.autograd.Variable(torch.randn(64, self._n_latent)).to(self._device)

        self._writer = SummaryWriter(log_dir=self._model_path)

    @abc.abstractmethod
    def _d_loss(self, real_logits, fake_logits, real_images, fake_images):
        raise NotImplementedError()

    @abc.abstractmethod
    def _g_loss(self, fake_logits):
        raise NotImplementedError()

    @abc.abstractmethod
    def _clip_weights(self):
        pass

    def train(self, train_loader, n_epochs=1, n_critic=8, interval=100):
        for epoch in range(n_epochs):
            d_loss = g_loss = 0.0
            for i, data in enumerate(tqdm(train_loader)):
                img = data.to(self._device)
                batch_size = img.shape[0]

                # Train D
                self._discriminator.train()
                self._discriminator.zero_grad()
                z = torch.autograd.Variable(torch.randn(batch_size, self._n_latent)).to(self._device)
                real_image = img
                real_logits = self._discriminator(real_image)

                fake_image = self._generator(z)
                fake_logits = self._discriminator(fake_image)

                d_loss = self._d_loss(real_logits, fake_logits, real_image, fake_image)

                d_loss.backward()
                self._d_optimizer.step()

                self._clip_weights()

                # Train G
                # train generator every n_critic steps
                if self._global_step % n_critic == 0:
                    self._generator.train()
                    z = torch.autograd.Variable(torch.randn(batch_size, self._n_latent)).to(self._device)
                    fake_image = self._generator(z)
                    fake_logits = self._discriminator(fake_image)
                    g_loss = self._g_loss(fake_logits)

                    self._generator.zero_grad()
                    g_loss.backward()
                    self._g_optimizer.step()

                if self._global_step % interval == 0:
                    self._generator.eval()
                    fake_images = (self._generator(self._z_fixed).data + 1) / 2
                    self._writer.add_scalar('d_loss', d_loss.item(), self._global_step)
                    self._writer.add_scalar('g_loss', g_loss.item(), self._global_step)
                    self._writer.add_image('fake_image', torchvision.utils.make_grid(fake_images), self._global_step)

                self._global_step += 1

            print(f'Discriminator Loss: {d_loss.data:.4f}, Generator Loss: {g_loss.data:.4f}')

            if self._model_path:
                torch.save(self._generator.state_dict(), os.path.join(self._model_path, 'generator.pth'))
                torch.save(self._discriminator.state_dict(), os.path.join(self._model_path, 'discriminator.pth'))

class VanillaGANTrainer(GANTrainer):

    def __init__(self, generator, discriminator, g_optimizer, d_optimizer, criterion,
                 n_latent=100, device='cpu', writer=None, model_path=None, output_dir=None,
                 ):
        super().__init__(generator, discriminator, g_optimizer, d_optimizer,
                         n_latent, device, writer, model_path, output_dir)
        self._criterion = criterion

    def _d_loss(self, real_logits, fake_logits, real_images, fake_images):
        real_labels = torch.ones(real_logits).to(self._device)
        real_loss = self._criterion(real_logits, real_labels)
        fake_labels = torch.zeros(fake_logits).to(self._device)
        fake_loss = self._criterion(fake_logits, fake_labels)
        return real_loss + fake_loss

    def _g_loss(self, fake_logits):
        fake_labels = torch.ones(fake_logits).to(self._device)
        return self._criterion(fake_logits, fake_labels)

class WGANTrainer(GANTrainer):

    def __init__(self, generator, discriminator, g_optimizer, d_optimizer,
                 n_latent=100, device='cpu', writer=None, model_path=None, output_dir=None, clip_value=0.01):
        super().__init__(generator, discriminator, g_optimizer, d_optimizer,
                         n_latent, device, writer, model_path, output_dir)
        self._clip_value = clip_value

    def _d_loss(self, real_logits, fake_logits, real_images, fake_images):
        return -torch.mean(real_logits) + torch.mean(fake_logits)

    def _g_loss(self, fake_logits):
        return -torch.mean(fake_logits)

    def _clip_weights(self):
        for p in self._discriminator.parameters():
            p.data.clamp_(-self._clip_value, self._clip_value)

class GPGANTrainer(GANTrainer):

    def __init__(self, generator, discriminator, g_optimizer, d_optimizer,
                 n_latent=100, device='cpu', writer=None, model_path=None, output_dir=None, gp_weight=10,
                 ):
        super().__init__(generator, discriminator, g_optimizer, d_optimizer,
                         n_latent, device, writer, model_path, output_dir)
        self._gp_weight = gp_weight

    def _gradient_penalty(self, real, fake):
        batch_size = real.shape[0]
        # choose random interpolation point
        alpha = torch.rand(batch_size, 1, 1, 1).to(self._device)
        interpolates = (alpha * real + ((1 - alpha) * fake)).requires_grad_(True)
        # calculate probability of interpolates
        interpolates_prob = self._discriminator(interpolates)
        # calculate gradients of probabilities with respect to interpolates
        fake = torch.autograd.Variable(torch.ones(interpolates_prob.size()), requires_grad=True).to(self._device)
        gradients = torch.autograd.grad(outputs=interpolates_prob, inputs=interpolates,
                                        grad_outputs=fake, create_graph=True, retain_graph=True, only_inputs=True)[0]
        gradients = gradients.view(batch_size, -1)
        gradient_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean()
        return gradient_penalty

    def _d_loss(self, real_logits, fake_logits, real_images, fake_images):
        gradient_penalty = self._gp_weight * self._gradient_penalty(real_images, fake_images)
        return -torch.mean(real_logits) + torch.mean(fake_logits) + gradient_penalty

    def _g_loss(self, fake_logits):
        return -torch.mean(fake_logits)


import tensorflow as td
from tensorflow.keras.layers import Dense,Flatten,Conv2D,MaxPooling2D
import pickle
from tensorflow.keras.models import Sequential
from tensorflow.keras.callbacks import TensorBoard
import time
import os
import numpy as np

NAME='Dog VS CAT_{}'.format(str(time.time()))
print(str(time.time()))

tensorboard=TensorBoard(log_dir='logs\{}'.format(NAME))

print(os.getcwd())
X_in=open('X.pickle','rb')
X=pickle.load(X_in)
X_in.close()

Y_in=open('Y.pickle','rb')
Y=pickle.load(Y_in)
Y_in.close()

X=X/255.0

model=Sequential()

model.add(Conv2D(64,(3,3),input_shape=X.shape[1:],activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))

model.add(Conv2D(64,(3,3),activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))

model.add(Flatten())

model.add(Dense(64,activation='relu'))
model.add(Dense(1,activation='sigmoid'))

model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy']
              )
Y=np.array(Y)
model.fit(X,Y,epochs=3,validation_split=0.2,callbacks=[tensorboard])


from django.contrib.auth.tokens import default_token_generator
from django.core.mail import send_mail
from django.db.models import Avg
from django.shortcuts import get_object_or_404
from django_filters.rest_framework import DjangoFilterBackend
from rest_framework import filters, permissions, status, viewsets
from rest_framework.decorators import action, api_view, permission_classes
from rest_framework.response import Response
from rest_framework_simplejwt.tokens import AccessToken
from reviews.models import Category, Genre, Review, Title
from users.models import User

from .filters import TitleFilter
from .mixins import MasterViewSet
from .permissions import (IsAdminModerAuthor, IsAdminOrSuper,
                          IsAdminUserOrReadOnly)
from .serializers import (CategorySerializer, CommentSerializer,
                          ConfirmationCodeSerializer, GenreSerializer,
                          ReviewSerializer, SignUpSerializer,
                          TitleReadSerializer, TitleSerializer,
                          UserMeSerializer, UsersSerializer)

@api_view(['POST'])
@permission_classes([permissions.AllowAny])
def sign_up(request):
    serializer = SignUpSerializer(data=request.data)
    if serializer.is_valid(raise_exception=True):
        email = serializer.validated_data['email']
        username = serializer.validated_data['username']
        user, create = User.objects.get_or_create(
            email=email,
            username=username
        )
        confirmation_code = default_token_generator.make_token(user)
        user.confirmation_code = confirmation_code
        user.save()
        send_mail(
            'Введите код для продолжения регистрации',
            confirmation_code,
            'admin@gmail.com',
            [email],
            fail_silently=False,
        )
        return Response(serializer.data, status=status.HTTP_200_OK)
    return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

@api_view(['POST'])
@permission_classes([permissions.AllowAny])
def getting_token(request):
    serializer = ConfirmationCodeSerializer(data=request.data)
    if not serializer.is_valid(raise_exception=True):
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
    username = serializer.validated_data['username']
    confirmation_code = serializer.validated_data['confirmation_code']
    user = get_object_or_404(User, username=username)
    if not default_token_generator.check_token(user, confirmation_code):
        return Response(serializer.errors,
                        status=status.HTTP_400_BAD_REQUEST)
    token = str(AccessToken.for_user(user))
    content = {'token': token}
    return Response(content, status=status.HTTP_201_CREATED)

class UsersViewSet(viewsets.ModelViewSet):
    queryset = User.objects.all()
    serializer_class = UsersSerializer
    permission_classes = (permissions.IsAuthenticated, IsAdminOrSuper,)
    filter_backends = (filters.SearchFilter,)
    filterset_fields = ('username',)
    search_fields = ('username',)
    lookup_field = 'username'

    @action(
        methods=['GET', 'PATCH'],
        detail=False,
        url_path='me',
        permission_classes=(permissions.IsAuthenticated,)
    )
    def me(self, request):
        user = get_object_or_404(User, username=self.request.user)
        if request.method == 'GET':
            serializer = UserMeSerializer(user)
            return Response(serializer.data, status=status.HTTP_200_OK)
        if request.method == 'PATCH':
            serializer = UserMeSerializer(user,
                                          data=request.data,
                                          partial=True)
            serializer.is_valid(raise_exception=True)
            serializer.save()
            return Response(serializer.data, status=status.HTTP_200_OK)

class CategoryViewSet(MasterViewSet):
    """Viewset для объектов модели Category."""
    queryset = Category.objects.all()
    serializer_class = CategorySerializer

    @action(
        methods=['DELETE'],
        detail=False,
        url_path=r'(?P<slug>[-a-zA-Z0-9_]+)',
        url_name='delete_category'
    )
    def delete_category(self, request, slug):
        category = get_object_or_404(Category, slug=slug)
        category.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)

class GenreViewSet(MasterViewSet):
    """Viewset для объектов модели Genre."""
    queryset = Genre.objects.all()
    serializer_class = GenreSerializer

    @action(
        methods=['DELETE'],
        detail=False,
        url_path=r'(?P<slug>[-a-zA-Z0-9_]+)',
        url_name='delete_genre'
    )
    def delete_genre(self, request, slug):
        genre = get_object_or_404(Genre, slug=slug)
        genre.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)

class TitleViewSet(viewsets.ModelViewSet):
    """Viewset для объектов модели Title."""
    queryset = Title.objects.annotate(rating=Avg('reviews__score'))
    serializer_class = TitleSerializer
    filter_backends = (DjangoFilterBackend,)
    filterset_class = TitleFilter
    permission_classes = (IsAdminUserOrReadOnly,)

    def get_serializer_method(self):
        if self.request.method == 'GET':
            return TitleReadSerializer
        return TitleSerializer

class CommentViewSet(viewsets.ModelViewSet):
    """ViewSet для объектов модели Comment."""
    serializer_class = CommentSerializer
    permission_classes = (IsAdminModerAuthor,)

    def get_review(self):
        review_id = self.kwargs.get('review_id')
        return get_object_or_404(Review, pk=review_id)

    def get_queryset(self):
        return self.get_review().comments.all()

    def perform_create(self, serializer):
        serializer.save(author=self.request.user, review=self.get_review())

class ReviewViewSet(viewsets.ModelViewSet):
    """ViewSet для объектов модели Review."""
    serializer_class = ReviewSerializer
    permission_classes = (IsAdminModerAuthor,)

    def get_title(self):
        title_id = self.kwargs.get('title_id')
        return get_object_or_404(Title, pk=title_id)

    def get_queryset(self):
        return self.get_title().reviews.all()

    def perform_create(self, serializer):
        user = self.request.user
        serializer.save(author=user, title=self.get_title())


class Solution:
    def interpret(self, command):
        command = list(command)
        result = []
        for i in range(0, len(command)):
            if command[i] == 'G':
                result.append('G')
            elif command[i] == '(':
                if command[i+1] == ')':
                    result.append('o')
                else:
                    result.append('al')
            elif command[i] == '/':
                result.append('/')
        return ''.join(result)

if __name__ == '__main__':
    command = "G()(al)"
    s = Solution()
    print(s.interpret(command))


import geopandas as gpd
from matplotlib import pyplot as plt
get_ipython().magic('matplotlib inline')
get_ipython().magic("run '/Users/fcc/Documents/digital-connector-python/recipe.py'")
dc_dir = '/Documents/TomboloDigitalConnector'

# import geopandas as gpd
# import os
# from pathlib import Path
# from matplotlib import pyplot as plt

# home_dir = str(Path.home())

# %matplotlib inline
# %run os.path.join(home_dir, 'Desktop/python_library_dc/digital-connector-python/recipe.py')
# dc_dir = '/Desktop/TomboloDigitalConnector'

### recipe's subject

# Notice that we are using two more subjects here. These will serve different purposes.
# We will use subject_la to subset the road network for the borough of Islington. We will use an intersect spatial operation
# to do that.
subject_la = Subject(subject_type_label='localAuthority',provider_label='uk.gov.ons',
                  match_rule=Match_Rule(attribute_to_match_on="name", pattern="Islington"))

# This is our road network subject
subject = Subject(subject_type_label='space_syntax',provider_label='com.spacesyntax',
                  geo_match_rule=Geo_Match_Rule(geo_relation="intersects", subjects=[subject_la]))

subject_lsoa = Subject(subject_type_label='lsoa',provider_label='uk.gov.ons')

# subject_geometry_dft fetches the traffic counter locations represented as points
subject_geometry_dft = Subject(provider_label='uk.gov.dft',
                               subject_type_label='trafficCounter')

### recipe's datasources

la = Datasource(importer_class='uk.org.tombolo.importer.ons.OaImporter',
                            datasource_id='localAuthority')

openmap = Datasource(importer_class='uk.org.tombolo.importer.spacesyntax.OpenMappingImporter',
                            datasource_id='SpaceSyntaxOpenMapping')

trafficCounts = Datasource(importer_class='uk.org.tombolo.importer.dft.TrafficCountImporter',
                           datasource_id='trafficCounts',
                           geography_scope = ["London"]) ## Note that geography scope is specific

### First, lets get our attributes

countPedalCycles_attribute = AttributeMatcher(provider='uk.gov.dft',
                                     label='CountPedalCycles')

countCarTaxis_attribute = AttributeMatcher(provider='uk.gov.dft',
                                     label='CountCarsTaxis')

integration_2km = AttributeMatcher(label='integration2km',provider='com.spacesyntax')

### DC fields

# A very basic field that can handle numeric and time series attributes is LatestValueField. This essentially
# fetches the latest value within a time series if a time series exists. If not, it will just fetch the default value.
integration_2km_f = LatestValueField(attribute_matcher=integration_2km,
                                           label = 'Integration 2km')

count_pedal_cycles_f = LatestValueField(attribute_matcher=countCarTaxis_attribute,
                                          label='count_pedal_cycles')

count_car_taxis_f = LatestValueField(attribute_matcher=countPedalCycles_attribute,
                                          label='count_car_taxis')

# Next we need to assign the traffic counts to our subject
# As DfT traffic count geometry is points (the traffic count sensor location) we need to assign it to the nearest
# road segment. For that we use MapToNearestSubjectField

m_count_pedal_cycles_f = MapToNearestSubjectField(field=count_pedal_cycles_f,
                                        label='Pedal traffic count',
                                        subject = subject_geometry_dft,
                                        max_radius = 1.)

m_count_car_taxis_f = MapToNearestSubjectField(field=count_car_taxis_f,
                                        label='Car/Taxi traffic count',
                                        subject = subject_geometry_dft,
                                        max_radius = 1.)
### running DC
dataset = Dataset(subjects=[subject],
                  fields=[integration_2km_f,
                          m_count_pedal_cycles_f,
                          m_count_car_taxis_f],
                  datasources=[la,
                               openmap,
                               trafficCounts])

recipe = Recipe(dataset,timestamp=False)
recipe.build_recipe(console_print=False)

recipe.run_recipe(tombolo_path='/Users/fcc/Documents/TomboloDigitalConnector',
                  clear_database_cache=False,
                  output_path = 'Documents/test.json')

gdf = gpd.read_file("/Users/fcc/Documents/test.json")
gdf.head()

vmin=gdf['Integration 2km'].min()
vmax=gdf['Integration 2km'].max()

ax = gdf.plot(column='Integration 2km', cmap='viridis',
              vmin=vmin,
              vmax=vmax)

# add colorbar
fig = ax.get_figure()
cax = fig.add_axes([0.9, 0.1, 0.03, 0.8])
sm = plt.cm.ScalarMappable(cmap='viridis', norm=plt.Normalize(vmin=vmin, vmax=vmax))
sm._A = []
fig.colorbar(sm, cax=cax)


import smtplib, ssl
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from vocabulary_extractor import *
import os

# Pointing path for crontab so that it sees .docx
abspath = os.path.abspath(__file__)
dname = os.path.dirname(abspath)
os.chdir(dname)

CHOSEN_WORDS = choose_random_words(DOCUMENT_NAME, 2)

sender_email = "learn.python.web.programming@gmail.com"
receiver_email = "katharina.education@gmail.com"

message = MIMEMultipart("alternative")
message["Subject"] = "Vocabulary to learn!"
message["From"] = sender_email
message["To"] = receiver_email

# Create the plain-text and HTML version of your message
text = f"""\
Hi,
Here are the words to learn for today:\n
{CHOSEN_WORDS[0]}
{CHOSEN_WORDS[-1]}\n
Have a nice day!\n
***This message was sent using Python automation program :)***
"""

html = f"""\

Hi,

Here are the words to learn for today:

{CHOSEN_WORDS[0]}

{CHOSEN_WORDS[-1]}
\n

        Have a nice day!

    ***This message was sent using Python automation program :)***

"""

# Turn these into plain/html MIMEText objects
part1 = MIMEText(text, "plain")
part2 = MIMEText(html, "html")

# Add HTML/plain-text parts to MIMEMultipart message
# The email client will try to render the last part first
message.attach(part1)
message.attach(part2)

# Create secure connection with server and send email
context = ssl.create_default_context()

with smtplib.SMTP_SSL("smtp.gmail.com", 465, context=context) as server:
    server.login(sender_email, "password")
    server.sendmail(
        sender_email, receiver_email, message.as_string()
    )
    print("Email was sent!")


import numpy as np
import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.optim as optim
import torch.nn.functional as F
from model import Net

class Controller(nn.Module):

    def __init__(self, num_actions=10, hidden_size=64):
        super(Controller, self).__init__()

        self.cell = nn.GRUCell(
            input_size=num_actions,
            hidden_size=hidden_size
        )

        self.fc = nn.Linear(
            in_features=hidden_size,
            out_features=num_actions
        )

        self.num_actions = num_actions
        self.hidden_size = hidden_size
        self.epsilon = 0.8
        self.gamma = 1.0
        self.beta = 0.01
        self.max_depth = 6
        self.clip_norm = 0
        self.log_probs = []
        self.actions = []
        self.entropies = []
        self.reward = None

        self.index_to_action = {
            0: 1,
            1: 2,
            2: 4,
            3: 8,
            4: 16,
            5: 'Sigmoid',
            6: 'Tanh',
            7: 'ReLU',
            8: 'LeakyReLU',
            9: 'EOS'
        }

        self.optimizer = optim.Adam(self.parameters(), lr=1e-2)

    def forward(self, x, h):
        x = x.unsqueeze(dim=0)
        h = h.unsqueeze(dim=0)

        h = self.cell(x, h)
        x = self.fc(h)

        x = x.squeeze(dim=0)
        h = h.squeeze(dim=0)

        return x, h

    def step(self, state):
        logits, new_state = self(torch.zeros(self.num_actions), state)

        idx = torch.distributions.Categorical(logits=logits).sample()
        probs = F.softmax(logits, dim=-1)
        log_probs = torch.log(probs)

        action = self.index_to_action[int(idx)]
        self.actions.append(action)

        entropy = -(log_probs * probs).sum(dim=-1)
        self.entropies.append(entropy)

        if action == 'EOS' and len(self.actions) <= 2:
            self.reward -= 1
        elif len(self.actions) >= 2 and isinstance(self.actions[-1], int) and isinstance(self.actions[-2], int):
            self.reward -= 0.1
        elif len(self.actions) >= 2 and isinstance(self.actions[-1], str) and isinstance(self.actions[-2], str) and action != 'EOS':
            self.reward -= 0.1

        terminate = action == 'EOS' or len(self.actions) == self.max_depth

        return log_probs[idx], new_state, terminate

    def generate_rollout(self, iter_train, iter_dev, verbose=False):
        self.log_probs = []
        self.actions = []
        self.entropies = []
        self.reward = None

        state = torch.zeros(self.hidden_size)
        terminated = False
        self.reward = 0

        while not terminated:
            log_prob, state, terminated = self.step(state)
            self.log_probs.append(log_prob)

        if verbose:
            print('\nGenerated network:')
            print(self.actions)

        net = Net(self.actions)
        accuracy = net.fit(iter_train, iter_dev)
        self.reward += accuracy

        return self.reward

    def optimize(self):
        G = torch.ones(1) * self.reward
        loss = 0

        for i in reversed(range(len(self.log_probs))):
            G = self.gamma * G
            loss = loss - (self.log_probs[i]*Variable(G)) - self.beta * self.entropies[i]

        loss /= len(self.log_probs)

        self.optimizer.zero_grad()
        loss.backward()

        if self.clip_norm > 0:
            nn.utils.clip_grad_norm_(self.parameters(), self.clip_norm)

        self.optimizer.step()

        return float(loss.data.numpy())


import sys
sys.path.append('D:\\These Clément\\these\\python_clément')
sys.path.append('/home/zouzou/these/python_clément')
from analyse import *

data=np.zeros((11,11))
for i in range(11) :
	for j in range(11) :
		x,y=extract_data('x=%i.000000,y=%i.000000'%(i,j))
		data[i,j]=max(y)
		xmin=73
		xmax=85
		x=x[xmin:xmax]
		y=y[xmin:xmax]
		[a,b],yfit=lin_fit(x,y)

data[10,10]=np.NaN
print_map(data)

# x,y=extract_data('x=%i.000000,y=%i.000000'%(0,0))
# plt.plot(x,y,'x-')

# xmin=73
# xmax=85
# x=x[xmin:xmax]
# y=y[xmin:xmax]
# [a,b],yfit=lin_fit(x,y)
# print(-b/a,a)
# plt.plot(x,yfit)

# xmin=437
# xmax=449
# x=x[xmin:xmax]
# y=y[xmin:xmax]
# [a,b],yfit=lin_fit(x,y)
# print(-b/a,a)
# plt.plot(x,yfit)

# x,y=extract_data('x=%i.000000,y=%i.000000'%(1,0))
# plt.plot(x,y,'x-')

plt.show()

data/20211118/Série ESR 100 modul f/traitement.py


def metade(valor=0, form=False):
    novo_valor = valor / 2
    if form == True:
        return moeda(novo_valor)
    else:
        return novo_valor

def dobro(valor=0, form=False):
    novo_valor = valor * 2
    if form == True:
        return moeda(novo_valor)
    else:
        return novo_valor

def aumentar(valor=0, p=0, form=False):
    novo_valor = valor + ((valor * p) / 100)
    if form == True:
        return moeda(novo_valor)
    else:
        return novo_valor

def diminuir(valor=0, p=0, form=False):
    novo_valor = valor - ((valor * p) / 100)
    if form == True:
        return moeda(novo_valor)
    else:
        return novo_valor

def moeda(valor=0, moeda='RS'):
    novo_valor = (f'{moeda} {valor:.2f}'.replace('.', ','))
    return novo_valor

def resumo(valor=0, aumento=0, reducao=0):
    print(f'''{"-" * 35}
{"RESUMO DO VALOR":^35}
{"-" * 35}''')
    print(f'Preço analisado:{moeda(valor):>19}')
    print(f'Dobro do preço:{dobro(valor, True):>20}')
    print(f'Metade do preço:{metade(valor, True):>19}')
    print(f'{aumento}% de aumento:{aumentar(valor, aumento, True):>20}')
    print(f'{reducao}% de redução:{diminuir(valor, reducao, True):>20}')


import os
import socket
import netifaces
from queue import Queue
from threading import Thread
from scapy.all import srp, Ether, ARP, conf

# Get the IP address of the default gateway
gateway_ip = netifaces.gateways()['default'][netifaces.AF_INET][0]
print("_"*100+"\n")
print(f'Your gateway address is {gateway_ip}')
print("\n")

ip_range = '.'.join(gateway_ip.split('.')[:-1])

# Get the IP address of the local interface
interface_name = netifaces.gateways()['default'][netifaces.AF_INET][1]
interface_ip = netifaces.ifaddresses(interface_name)[netifaces.AF_INET][0]['addr']

# Print your IP address
print("_"*100+"\n")
print("Your IP address is:", interface_ip)
print("\n")

print("_"*100+"\n")
print("LIVE IP ADDRESSES ON YOUR SUBNET:")
print("\n")

def do_stuff(q):
    while True:
        ip=ip_range + "." + str(q.get())
        response = os.popen(f"ping  {ip} -c 1").read()
        if "1 packets received" in response:
            try:
                hostname = socket.gethostbyaddr(ip)[0]
                print(" {}ONLINE".format( hostname))
            except socket.error:
                hostname = "No HOST NAME "
                print(" {}{} ONLINE".format( hostname,ip))
        q.task_done()

q = Queue(maxsize=0)
num_threads = 100

for i in range(num_threads):
    worker = Thread(target=do_stuff, args=(q,))
    worker.setDaemon(True)
    worker.start()

for x in range(1,256):
    q.put(x)

q.join()

# Check to see if the queue is empty. if it is print 'Done!'
if q.qsize() == 0:
    print('Done!')


"""MiraAssemblyFormat class for the 'mira' format within Galaxy."""

from galaxy.datatypes.data import Text

class MiraAssemblyFormat(Text):
    """MIRA Assembly Format  data."""

    file_ext = "mira"

    def sniff(self, filename):
        """Determine if the file is a MIRA Assembly Format file.

        Note currently this only detects MIRA Assembly Format v2.0,
        as used in MIRA v3.9 and v4.0.

        It does not detect MIRA Assembly Format v1 as used in both
        MIRA v3.2 and v3.4.
        """
        h = open(filename)
        line = h.readline()
        if line.rstrip() != "@Version\t2\t0":
            h.close()
            return False
        line = h.readline()
        if line.rstrip() != "@Program\tMIRALIB":
            h.close()
            return False
        return True

    def merge(split_files, output_file):
        """Merge MIRA assembly files (not implemented).

        Merging multiple MIRA files is non-trivial and may not be possible...
        """
        if len(split_files) == 1:
            # For one file only, use base class method (move/copy)
            return Text.merge(split_files, output_file)
        if not split_files:
            raise ValueError(
                "No MIRA files to merge, %r, into %r" % (split_files, output_file)
            )
        raise NotImplementedError(
            "Merging MIRA Assembly Files has not been implemented"
        )

    merge = staticmethod(merge)

    def split(cls, input_datasets, subdir_generator_function, split_params):
        """Split a MIRA Assembly File (not implemented for now)."""
        if split_params is None:
            return None
        raise NotImplementedError("Can't yet split a MIRA Assembly Format file")

    merge = staticmethod(merge)


my_value = int(5)
value = int(input("Введи число and press Enter:"))
while value != my_value or value == my_value:
    if value > my_value:
        value = int(input("Введите число меньше"))
    elif value < my_value:
        value = int(input("Введите число больше"))
    else:
        print("ты угадал" + " " + str(5))
        print('Спасибо за игру !')
        break

str_ = input('Введите строку: ')

if len(str_) > 2:
    if str_[-3:] == 'ing':
        str_ += 'ly'
    else:
        str_ += 'ing'
print(str_)


fo = open("a.txt", "r")

print("Name of the file: ", fo.name)
line = fo.readlines()

def namata(n):

  for i in range(1, 11):
     print(str(int(n))+'x'+str(int(i))+'=' + str(int(n)*int(i)))
     k=str(int(n)*int(i))
     file= open("b.txt", "a")
     file.write(f'{int(n)} x {int(i)} = {k}\n')
     file.close()

  return k

for j in range(len(line)):
  namata(line[j])


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import unicode_literals

from io import open

from unitex import *

_LOGGER = logging.getLogger(__name__)

class FSAConstants:

    EPSILON = "<E>"

    DEPTH_FIRST_SEARCH = "dfs"
    BREADTH_FIRST_SEARCH = "bfs"

class Edge(object):

    def __init__(self, label, targets=None, source=None):
        self.__label = label

        self.__source = source

        self.__targets = targets
        if self.__targets is not None:
            self.__tids = set([target.get_id() for target in targets])

    def __len__(self):
        return len(self.__targets)

    def __str__(self):
        label = self.get_label()
        label = label.encode(UnitexConstants.DEFAULT_ENCODING)
        return label

    def __unicode__(self):
        return u"%s" % self.get_label()

    def __hash__(self):
        return hash(self.get_label())

    def __cmp__(self, e):
        return cmp(self.get_label(), self.get_label())

    def __iter__(self):
        for target in self.__targets:
            yield target

    def __contains__(self, target):
        return True if target.get_id() in self.__tids else False

    def __getitem__(self, i):
        return self.__targets[i]

    def get_label(self):
        return self.__label

    def get_source(self):
        return self.__source

    def set_source(self, source):
        self.__source = source

    def get_targets(self):
        return self.__targets

    def set_targets(self, targets):
        self.__targets = targets
        self.__tids = set([target.get_id() for target in targets])

    def add_target(self, target):
        if target.get_id() in self.__tids:
            return
        self.__targets.append(target)

    def del_target(self, target):
        if target.get_id() not in self.__tids:
            return

        self.__tids.remove(target.get_id())

        for i in range(len(self.__targets)):
            _target = self.__targets[i]
            if _target.get_id() == target.get_id():
                del self.__targets[i]
                break

class Node(object):

    def __init__(self, _id, final=False):
        self.__id = _id

        self.__final = final
        self.__edges = {}

        self.__depth = 0

        self.__visited = False

    def __len__(self):
        return len(self.__edges)

    def __contains__(self, label):
        return label in self.__edges

    def __getitem__(self, label):
        return self.__edges.get(label, None)

    def __iter__(self):
        for label in self.__edges:
            yield label

    def __str__(self):
        node = self.__unicode__()
        node = node.encode(UnitexConstants.DEFAULT_ENCODING)
        return node

    def __unicode__(self):
        s = u"NODE[%s]" % str(self.get_id())

        if self.is_final():
            s += u" -- FINAL"

        for label in self:
            targets = u" | ".join([str(target.get_id()) for target in self[label]])
            s += u"\n\t%s -> (%s)" % (label, targets)

        return s

    def get_id(self):
        return self.__id

    def set_id(self, i):
        self.__id = i

    def is_deterministic(self):
        if FSAConstants.EPSILON in self.__edges:
            return False

        for label in self.__edges:
            if len(self[label]) > 1:
                return False

        return True

    def exists(self, label, node=None):
        if not label in self:
            return False

        if node is not None and node not in self[label]:
            return False

        return True

    def add(self, label, target):
        if self.exists(label, target) is True:
            return

        if self.exists(label) is False:
            edge = Edge(label, [target], self)
            self.__edges[label] = edge
        else:
            self[label].add_target(target)

    def delete(self, label, node=None):
        if not self.exists(label, node):
            raise UnitexException("Edge not found: %s" % label)

        if node is None:
            del self.__edges[label]
        else:
            self[label].del_target(node)

    def set_depth(self, depth):
        self.__depth = depth

    def get_depth(self):
        return self.__depth

    def is_visited(self):
        return self.__visited

    def set_visited(self, visited=True):
        self.__visited = visited

    def is_final(self):
        return self.__final

    def set_final(self, final=True):
        self.__final = final

class NodeSets(object):

    def __init__ (self):
        self.__sets = {}

    def __getitem__(self, _id):
        return self.__sets[_id]

    def __contains__(self, s):
        return s in self.all()

    def __iter__ (self):
        return iter(self.all())

    def all(self):
        return set([tuple(l) for l in self.__sets.values()])

    def add(self, s):
        _set = tuple(sorted(set(s)))
        for _id in s:
            self.__sets[_id] = _set

class Automaton(object):

    def __init__(self, name="Automaton"):
        self.__name = name

        self.__nodes = []

        self.__initial = 0
        self.__finals = []

        self.__nodes.append(Node(self.__initial, False))

    def __len__(self):
        return len(self.__nodes)

    def __getitem__(self, _id):
        try:
            return self.__nodes[_id]
        except IndexError:
            return None

    def __iter__(self):
        for node in self.__nodes:
            yield node

    def __str__(self):
        automaton = self.__unicode__()
        automaton = automaton.encode(UnitexConstants.DEFAULT_ENCODING)
        return automaton

    def __unicode__(self):
        title = u"# FSA -- %s #" % self.get_name()

        s = u"%s\n%s\n%s\n\n" % ("#" * len(title), title, "#" * len(title))

        for node in self:
            s += u"%s\n\n" % node

        return s

    def get_name(self):
        return self.__name

    def set_name(self, name):
        self.__name = name

    def get_depth(self):
        depth = 0
        for nid in self.__finals:
            final = self.__nodes[nid]

            if final.get_depth() > depth:
                depth = final.get_depth()

        return depth

    def get_initial(self):
        return self.__initial

    def set_initial(self, initial):
        self.__initial = initial

    def get_finals(self):
        return self.__finals

    def set_finals(self, finals):
        self.__finals = finals

    def get_nodes(self):
        return self.__nodes

    def set_nodes(self, nodes):
        self.__nodes = nodes

    def add_edge(self, label, sid, tid):
        source = self[sid]
        target = self[tid]

        target.set_depth(source.get_depth() + 1)

        source.add(label, target)

    def add_node(self, initial=False, final=False):
        if initial is True:
            return self.__initial
        elif final is True:
            self.__finals.append(len(self.__nodes))
            self.__nodes.append(Node(self.__finals[-1], True))
            return self.__finals[-1]

        nid = len(self.__nodes)

        self.__nodes.append(Node(nid, final))

        return nid

    def add_path(self, path):
        if len(path) == 0:
            raise UnitexException("Empty path!")
        sid = self.add_node(initial=True, final=False)

        for label in path[:-1]:
            tid = self.add_node(initial=False, final=False)
            self.add_edge(label, sid, tid)

            sid = tid
        else:
            self.add_edge(path[-1], sid, self.add_node(initial=False, final=True))

    def get_alphabet(self):
        alphabet = set()

        for node in self:
            for label in node:
                alphabet.add(label)

        return tuple(alphabet)

    def is_deterministic(self):
        for node in self:
            if not node.is_deterministic():
                return False
        return True

    def __closure(self, nid):
        stack = [nid]
        result = set(stack)

        while len(stack) > 0:
            current = stack.pop()

            if FSAConstants.EPSILON in self[current]:
                edge = self[current][FSAConstants.EPSILON]
                if edge not in result:
                    stack.append(edge)
                    result.add(edge)

        return tuple(result)

    def determinize(self):
        dfa = Automaton("DETERMINIZED(%s)" % self.get_name())

        alphabet = self.get_alphabet()

        initials = self.__closure(self.get_initial())

        hid = dfa.add_node(initial=True, final=False)

        visited = {}
        visited[initials] = hid

        stack = [initials]
        while len(stack) > 0:
            current = stack.pop()

            for label in alphabet:
                new = set()
                for node in current:
                    if not label in self[node]:
                        continue
                    for next in self[node][label]:
                        new.update(self.__closure(next.get_id()))
                new = tuple(new)

                if len(new) == 0:
                    continue

                if new not in visited:
                    stack.append(new)

                    final = True in [self[_id].is_final() for _id in new]
                    nid = dfa.add_node(final=final)

                    visited[new] = nid

                dfa.add_edge(label, visited[current], visited[new])

        self.set_name(dfa.get_name())

        self.set_initial(dfa.get_initial())
        self.set_finals(dfa.get_finals())

        self.set_nodes(dfa.get_nodes())

    def minimize(self):
        min = Automaton("MINIMIZED(%s)" % self.get_name())

        alphabet = self.get_alphabet()

        nodetoset = {}
        settonode = {}

        sets = NodeSets()

        rest, final = [], []
        for node in self:
            if node.is_final():
                final.append(node.get_id())
            else:
                rest.append(node.get_id())

        sets.add(rest)
        sets.add(final)

        stack = [s for s in sets if len(s) > 1]

        def target_set(_id, label):
            edge = self[_id][label]

            if edge is None:
                return None
            else:
                return sets[edge[0].get_id()]

        while len(stack) > 0:
            current = stack.pop()

            for label in alphabet:
                target = target_set(current[0], label)

                one, two = [current[0]], []
                for _id in current[1:]:
                    if target_set(_id, label) == target:
                        one.append(_id)
                    else:
                        two.append(_id)

                if len(two) > 0:
                    sets.add(one)
                    sets.add(two)

                    if len(one) > 1:
                        stack.append(one)
                    if len(two) > 1:
                        stack.append(two)

                    break

        for s in sets:
            initial = self.get_initial() in s
            final = True in [self[_id].is_final() for _id in s]

            _id = min.add_node(initial=initial, final=final)

            nodetoset[_id] = s
            settonode[s] = _id

        for node in min:
            done = set()

            s = nodetoset[node.get_id()]

            source = self[s[0]]
            for label in source:
                edge = source[label]

                if label in done:
                    continue
                done.add(label)

                for target in edge:
                    t = sets[target.get_id()]
                    min.add_edge(label, node.get_id(), settonode[t])

        self.set_name(min.get_name())

        self.set_initial(min.get_initial())
        self.set_finals(min.get_finals())

        self.set_nodes(min.get_nodes())

    def reset(self):
        for node in self:
            node.set_visited(False)

    def __expand(self, source):
        L = []

        source.set_visited(True)
        for label in source:
            edge = source[label]
            for target in source[label]:
                L.append((edge.get_label(), source.get_id(), target.get_id()))

        return L

    def iter(self, iter_type=None):
        if iter_type is None:
            iter_type = FSAConstants.BREADTH_FIRST_SEARCH

        if len(self[self.get_initial()]) == 0:
            raise UnitexException("Empty FSA")

        i = None
        if iter_type == FSAConstants.DEPTH_FIRST_SEARCH:
            i = -1
        elif iter_type == FSAConstants.BREADTH_FIRST_SEARCH:
            i = 0
        else:
            raise UnitexException("Unknown iter type: %s" % iter_type)

        root = self[self.get_initial()]
        if root.is_visited():
            self.reset()

        L = self.__expand(root)
        while L:
            edge, sid, tid = L.pop(i)
            yield (edge, sid, tid)

            if not self[tid].is_visited():
                L += self.__expand(self[tid])

    def save(self, file, encoding=None):
        if encoding is None:
            encoding = UnitexConstants.DEFAULT_ENCODING

        with open(file, "w", encoding=encoding) as output:
            output.write("digraph Automaton {\n\n")
            output.write("\tcenter = 1;\n")
            output.write("\tcharset = \"%s\";\n" % encoding)
            output.write("\trankdir = LR;\n")
            output.write("\tranksep = 1;\n")
            output.write("\tedge [arrowhead = vee];\n\n")

            nodes = set()
            edges = set()

            for node in self:
                sid = node.get_id()
                n1 = "node%s" % sid

                if not sid in nodes:
                    nodes.add(sid)

                    if node.get_id() == self.get_initial():
                        output.write("\t%s[shape = circle, label = \"\"];\n" % n1)
                    elif node.is_final():
                        output.write("\t%s[shape = doublecircle, label = \"\"];\n" % n1)
                    else:
                        output.write("\t%s[shape = point, label = \"\"];\n" % n1)

                for label in node:
                    for target in node[label]:
                        if (node.get_id(), label, target.get_id()) in edges:
                            continue
                        edges.add((node.get_id(), label, target.get_id()))

                        tid = target.get_id()
                        n2 = "node%s" % tid

                        if not tid in nodes:
                            nodes.add(tid)

                            if target.get_id() == self.get_initial():
                                output.write("\t%s[shape = circle, label = \"\"];\n" % n2)
                            elif target.is_final():
                                output.write("\t%s[shape = doublecircle, label = \"\"];\n" % n2)
                            else:
                                output.write("\t%s[shape = point, label = \"\"];\n" % n2)

                        output.write("\t%s -> %s [label = \"%s\"];\n" % (n1, n2, label))

                output.write("\n")

            output.write("}\n")


# 珠玑妙算游戏（the game of master mind）的玩法如下。
#  计算机有4个槽，每个槽放一个球，颜色可能是红色（R）、黄色（Y）、绿色（G）或蓝色（B）。例如，计算机可能有RGGB 4种（槽1为红色，槽2、3为绿色，槽
# 4为蓝色）。作为用户，你试图猜出颜色组合。打个比方，你可能会猜YRGB。要是猜对某个槽的颜色，则算一次“猜中”；要是只猜对颜色但槽位猜错了，则算一次“伪猜中”。
# 注意，“猜中”不能算入“伪猜中”。
#  给定一种颜色组合solution和一个猜测guess，编写一个方法，返回猜中和伪猜中的次数answer，其中answer[0]为猜中的次数，answer[
# 1]为伪猜中的次数。
#  示例：
#  输入： solution="RGBY",guess="GGRR"
# 输出： [1,1]
# 解释： 猜中1次，伪猜中1次。
#
#  提示：
#
#  len(solution) = len(guess) = 4
#  solution和guess仅包含"R","G","B","Y"这4种字符
#
#  Related Topics 数组

# leetcode submit region begin(Prohibit modification and deletion)
from collections import Counter
class Solution:
    def masterMind(self, solution: str, guess: str) -> List[int]:
        dicts=Counter(solution)
        dictg=Counter(guess)
        answer=[0,0]
        for i in range(len(solution)):
            if solution[i]==guess[i]:
                answer[0]+=1
                dicts[solution[i]]-=1
                dictg[guess[i]]-=1
        for i in range(len(solution)):
            if guess[i] in solution and dicts[guess[i]]>0 and dictg[guess[i]]>0:
                answer[1]+=1
                dicts[guess[i]]-=1
                dictg[guess[i]]-=1
        return answer
# leetcode submit region end(Prohibit modification and deletion)


from torch.nn import Module
from torch import cat
import torch

from dupimage.helper import get_device, get_dataloader, ImageDataset

class BaseExtractor(Module):
    """
    Must override it

    model
    device
    root_dir
    dataset
    dataloader
    labels
    """
    def __init__(self, root_dir='./', suffix='jpg', batch_size=64, num_workers=1):
        super().__init__()
        self.device = get_device()
        self.root_dir = root_dir

        # Get Dataset and Dataloader objects from root_dir
        self.dataset = ImageDataset(root_dir=root_dir, suffix=suffix)
        self.dataloader = get_dataloader(self.dataset, batch_size, num_workers)

        # Get labels
        self.labels = [label.split('/')[-2] + '/' + label.split('/')[-1] for label in self.dataset.files]

    def get_feature_vectors(self, n_components=25088 + 6272):
        """

        """
        features = None

        # get features
        for batch in self.dataloader:
            if self.device != 'cpu':
                batch[0] = batch[0].to(self.device).float()
            output = self(batch[0])
            if features == None:
                features = output
            else:
                features = cat([features, output], dim=0)
            del batch
            torch.cuda.empty_cache()

        return features


from unittest import TestCase

from day10 import NavSubsystemParser, InvalidClosingTokenError, Day10, LineCompletion

EXAMPLE_INPUT = '''
[({(<(())[]>[[{[]{<()<>>
[(()[<>])]({[<{<<[]>>(
{([(<{}[<>[]}>{[]{[(<()>
(((({<>}<{<{<>}{[]{[]{}
[[<[([]))<([[{}[[()]]]
[{[{({}]{}}([{[{{{}}([]
{<[[]]>}<{[{[{[]{()[[[]
[<(<(<(<{}))><([]([]()
<{([([[(<>()){}]>(<<{{
<{([{{}}[<[[[<>{}]]]>[]]
'''.strip()

EXPECTED_SCORE_PART_ONE = 26397
EXPECTED_SCORE_PART_TWO = 288957

EXPECTED_INCOMPLETE_LINES = [
    '[({(<(())[]>[[{[]{<()<>>',
    '[(()[<>])]({[<{<<[]>>(',
    '(((({<>}<{<{<>}{[]{[]{}',
    '{<[[]]>}<{[{[{[]{()[[[]',
    '<{([{{}}[<[[[<>{}]]]>[]]'
]

class TestDay10(TestCase):

    def setUp(self) -> None:
        self.sut = Day10(EXAMPLE_INPUT.split("\n"))

    def test_part_one(self):
        self.assertEqual(EXPECTED_SCORE_PART_ONE, self.sut.part_one())

    def test_part_two(self):
        self.assertEqual(EXPECTED_SCORE_PART_TWO, self.sut.part_two())

    def test_only_incomplete_lines_length(self):
        lines = self.sut.only_incomplete_lines()
        print(lines)
        self.assertEqual(5, len(lines))
        lines = [str(line) for line in lines]
        self.assertListEqual(sorted(EXPECTED_INCOMPLETE_LINES), sorted(lines))

    def test_completion_scoring(self):
        self.assertEqual(288957, self.sut.complete_score('}}]])})]'))
        self.assertEqual(5566, self.sut.complete_score(')}>]})'))
        self.assertEqual(1480781, self.sut.complete_score('}}>}>))))'))

class TestNavSubsystemParserChunks(TestCase):

    def test_flattening_single_incomplete_chunk_to_string(self):
        input_str = '['
        sut = NavSubsystemParser.Chunk(input_str)

        self.assertEqual(input_str, str(sut))

    def test_flattening_single_closed_chunk_to_string(self):
        opening_char = '['
        closing_char = ']'
        sut = NavSubsystemParser.Chunk(opening_char)
        sut.close(closing_char)

        self.assertEqual(opening_char + closing_char, str(sut))

    def test_flattening_nested_incomplete_chunks_to_string(self):
        root_opening = '['
        child_opening = '<'
        grand_child_opening = '('

        sut = NavSubsystemParser.Chunk(root_opening)
        child = NavSubsystemParser.Chunk(child_opening)
        grand_child = NavSubsystemParser.Chunk(grand_child_opening)
        child.add_child(grand_child)
        sut.add_child(child)

        self.assertEqual(root_opening + child_opening + grand_child_opening, str(sut))

    def test_flattening_nested_partially_complete_chunks_to_string(self):
        root_opening = '['
        child_opening = '<'
        grand_child_opening = '('
        grand_child_closing = ')'

        sut = NavSubsystemParser.Chunk(root_opening)
        child = NavSubsystemParser.Chunk(child_opening)
        grand_child = NavSubsystemParser.Chunk(grand_child_opening)
        grand_child.close(grand_child_closing)
        child.add_child(grand_child)
        sut.add_child(child)

        self.assertEqual(root_opening + child_opening + grand_child_opening + grand_child_closing, str(sut))

    def test_flattening_example_comple_line(self):
        input_str = '[({(<(())[]>[[{[]{<()<>>'
        self.assertEqual(input_str, str(NavSubsystemParser.parse_line(input_str)))

class TestNavSubsystemParser(TestCase):

    def test_parse_line_with_several_root_chunks(self):
        parsed_line = NavSubsystemParser.parse_line('()[]<>{[]}')
        self.assertEqual(4, len(parsed_line.chunks))

    def test_parse_line_fails_with_example_1(self):
        self.assertRaises(InvalidClosingTokenError, NavSubsystemParser.parse_line, '{([(<{}[<>[]}>{[]{[(<()>')

    def test_parse_line_failure_reports_curly_bracket(self):
        expected_token = '}'
        try:
            NavSubsystemParser.parse_line('{([(<{}[<>[]}>{[]{[(<()>')
        except InvalidClosingTokenError as error:
            print(error)
            self.assertEqual(expected_token, error.token)
        else:
            self.fail('Should raise InvalidClosingTokenError')

class TestLineCompletion(TestCase):
    def test_complete_only_root(self):
        incomplete_root = '{'
        parsed_line = NavSubsystemParser.parse_line(incomplete_root)

        sut = LineCompletion(parsed_line.chunks)
        self.assertEqual('}', sut.complete())

    def test_nested_chunks(self):
        incomplete_line = '[<<>{('
        parsed_line = NavSubsystemParser.parse_line(incomplete_line)

        sut = LineCompletion(parsed_line.chunks)
        self.assertEqual(')}>]', sut.complete())


from typing import Iterable

from dateutil.rrule import rrulestr
from django.db import models, transaction as db_transaction
from django.utils import timezone
from django.utils.translation import gettext_lazy as _

from ..models import BaseModel
from ..settings import Settings
from .validators import validate_rrule

class BaseTransaction(BaseModel):

    class Meta:
        abstract = True

    timestamp = models.DateField(_('timestamp'), default=timezone.now)
    amount = models.BigIntegerField(_('amount')) if Settings.USE_BIG_INTEGER else models.IntegerField(_('amount'))
    description = models.TextField(_('description'))

    # TODO: consider adding currency support (e.g. local_amount and local_currency)

    def __str__(self):
        return self.description

class RecurringTransaction(BaseTransaction):

    class Meta:
        verbose_name = _('recurring transaction')
        verbose_name_plural = _('recurring transactions')

    recurrence = models.TextField(_('recurrence'), validators=[validate_rrule])
    last_occurred_at = models.DateTimeField(blank=True)

    entity = models.ForeignKey(Settings.FINANCIAL_ENTITY_MODEL, verbose_name=_('entity'), related_name='recurring_transactions', on_delete=models.PROTECT)

    @property
    def recurrence_rule(self):
        # Parse recurrence rule
        return rrulestr(self.recurrence, cache=True)

    @staticmethod
    def generate_transactions(recurring_transactions: Iterable['RecurringTransaction'] = None):
        with db_transaction.atomic():
            # Find recurring transactions if not provided
            if not recurring_transactions:
                recurring_transactions = RecurringTransaction.objects.all()

            for recurring_transaction in recurring_transactions:
                # Parse recurrence rule and determine occurrences between the last occurrence and now
                if recurring_transaction.last_occurred_at:
                    occurrences = recurring_transaction.recurrence_rule.between(recurring_transaction.last_occurred_at, timezone.now(), inc=False)
                else:
                    occurrences = recurring_transaction.recurrence_rule.before(timezone.now(), inc=False)

                for occurrence in occurrences:
                    # TODO: replace placeholders in description (date, etc.)

                    # Create transaction
                    transaction = Transaction(timestamp=occurrence, amount=recurring_transaction.amount, description=recurring_transaction.description,
                                              entity=recurring_transaction.entity)
                    transaction.save()

                    # Update last occurrence
                    recurring_transaction.last_occurred_at = occurrence

                # Save last occurrence
                recurring_transaction.save()

class Transaction(BaseTransaction):

    class Meta:
        verbose_name = _('transaction')
        verbose_name_plural = _('transactions')

    entity = models.ForeignKey(Settings.FINANCIAL_ENTITY_MODEL, verbose_name=_('entity'), related_name='transactions', on_delete=models.PROTECT)
    recurring_transaction = models.ForeignKey(RecurringTransaction, verbose_name=_('recurring transaction'), related_name='transactions',
                                              blank=True, null=True, on_delete=models.SET_NULL)

    if Settings.TRANSACTION_SETTLEMENT_ENABLED:
        settled_by = models.ForeignKey('self', verbose_name=_('settled by'), related_name='settled', blank=True, null=True, on_delete=models.SET_NULL)

    @staticmethod
    def generate_settlements():
        with db_transaction.atomic():
            # Find entities and their balances
            entities = Settings.get_financial_entity_model().objects.with_balance().all()

            for entity in entities:
                # Find unsettled transactions
                transactions = entity.transactions.filter(settled_by__isnull=True, settled__isnull=True)

                # Sanity check for current state
                if entity.balance != sum([transaction.amount for transaction in transactions]):
                    raise Exception('Balance and sum of unsettled transactions are not equal.')

                # Check if there is a balance to settle
                if entity.balance == 0:
                    continue

                # Create settlement transaction
                settlement_transaction = Transaction(description=Settings.TRANSACTION_SETTLEMENT_DESCRIPTION, amount=-entity.balance, entity=entity)
                settlement_transaction.save()
                settlement_transaction.settled.set(transactions)


# -*- coding: utf-8 -*-
import image
import scrapy
import re
from scrapy.shell import inspect_response  # for debugging
from scrapy.http import Request
import os, sys; sys.path.append(os.path.dirname(os.path.realpath('webdriver.py')))
from . import webdriver

class restaurantInfo:
    def __init__(self, basic_data, advanced_data):
        self.basic_d = basic_data
        self.advanced_d = advanced_data

class TalabatbotSpider(scrapy.Spider):
    name = 'talabatBot'
    mainDomain: str = 'https://www.talabat.com'
    start_urls = ["https://www.talabat.com/qatar/restaurants"]
    default_location = "al-mansoura"
    custom_settings = {
        'FEED_URI': 'talabat.json'
    }
    item_num = 0
    possibleLocations = ['ain-khaled?aid=1740', 'onaiza?aid=1700']
    restaurant_advanced_info = []
    dataList = []

    # Data processing methods
    def fix_data_file(self, data):
        data['type'] = data['@type']
        data['context'] = data['@context']
        del data['@type']
        del data['@context']
        del data['@id']
        del data['image']

    def fix_JSON_format(self, obj):
        new_json = {}   # put menu section after it was modified in here
        new_json['menuSections'] = []
        new_json['menu_id'] = obj['result']['menu']['id']

        menuSection = obj['result']['menu']['menuSection'] # a list of menu sections
        for section in menuSection:
            new_json['menuSections'].append({'sectionName': section['nm'], 'items': self.fix_item_list(section['itm']) })

        return new_json

    def fix_item_list(self, item_list):
        new_item_list = []
        for item in item_list:
            new_item = {'name'   : item['nm'],
                        'itemID' : item['id'],
                        'rating' : item['rt'],
                        'price'  : item['pr'],
                        'image'  : item['img'],
                        'description' : item['dsc']}
            new_item_list.append(new_item)

        return new_item_list

    # Parse is called whenever the spider successfully crawls a URL
    # Response object is automatically filled with page info and passed here
    def parse(self, response):
        """
        Step 1: Main parse callback, crawling starts here "talabat/qatar/allRestaurants"
        Gather: None
        Fetch : All restaurant links
        """

        restaurant_list = webdriver.runScrollDriver(self.start_urls[0])
        for restaurant in restaurant_list:
            yield scrapy.Request(f"{self.mainDomain}/qatar/{restaurant}", callback=self.parse_restaurant_page)

    def parse_restaurant_page(self, response):
        """
        Step 2: Parsing individual restaurant pages "talabat/qatar/specificRestaurant"
        Gather: basic info (e.g Restaurant Name)
        Fetch : Link to Menu page
        """

        # 1) Get the json file containing all restaurant information
        try:
            # Get the basic data of restaurant
            self.data = self.get_JSON_File(response)[0]
            if (self.data['@type'] != 'Restaurant' and self.data['@type'] != 'restaurant'):
                return {response.url : 'Not restaurant'}
            self.fix_data_file(self.data)
            self.dataList.append(self.data)
            restaurant_ID = re.findall("bid:\d\d\d\d\d", response.css("script").extract()[22])[0][4:]
            valid_locations = re.findall("Al \w+", self.data['description'])
            if (len(valid_locations) == 0):
                valid_locations = re.search("Umm \w+", self.data['description']).group()

            # 2) search for the mainModel JSON
            text = response.xpath("//script[@type='text/javascript']").extract()[8]
            result = re.search('id:\d\d\d\d,cid:\d\d,cn:"\w+",an:"\D+"', text).group()
            result = result.split('"')
            valid_location = result[-2].replace(" ", "-")
            aid = result[0][3:7]
            valid_location = valid_locations[0].rstrip().replace(" ", "-")
        except:
            self.item_num += 1
            while (len(self.restaurant_advanced_info) < len(self.dataList)):
                self.restaurant_advanced_info.append(None)
            return {response.url : self.data}

        # 3) Get URL to menu page
        menu_url = f"https://www.talabat.com/qatar/restaurant/{restaurant_ID}/{valid_location}?aid={aid}"
        return scrapy.Request(menu_url, callback=self.parse_menu_page)

    def get_JSON_File(self, response):
        """ Method to get JSON file from a url """
        items_list = webdriver.runWebDriverJSON(response.url)
        return items_list

    # Step 3: Main parse, parse information from the restaurant
    # Parse Menu items / prices / working hours / etc
    def parse_menu_page(self, response):
        # Skip page if we were redirected to homepage from previous step
        if response.url == 'https://www.talabat.com/qatar':
            self.item_num += 1
            return {f"could not parse {self.dataList[-1]['url']}": None}
        self.restaurant_advanced_info.append(webdriver.runWebDriverMenuPage(response.url))
        self.restaurant_advanced_info[-1] = self.fix_JSON_format(self.restaurant_advanced_info[-1])
        self.item_num += 1
        if (self.restaurant_advanced_info[self.item_num-1] and self.dataList[self.item_num-1]):
            return { str(self.item_num-1): {**self.dataList[self.item_num-1], **self.restaurant_advanced_info[self.item_num-1]} }