import heapq
from typing import Optional

import numpy

Position = tuple[int, int]

class PriorityQueue:
    def __init__(self):
        self.elements: list[tuple[float, Position]] = []

    def empty(self) -> bool:
        return not self.elements

    def put(self, item: Position, priority: float) -> None:
        heapq.heappush(self.elements, (priority, item))

    def get(self) -> Position:
        return heapq.heappop(self.elements)[1]

def _parse_input(filename: str) -> numpy.ndarray:
    with open(f"advent21/inputs/{filename}", encoding="utf-8") as file:
        lines = file.readlines()
    return numpy.array([[int(number) for number in line.rstrip()] for line in lines])

def _heuristic(a: Position, b: Position) -> float:
    (x1, y1) = a
    (x2, y2) = b
    return abs(x1 - x2) + abs(y1 - y2)

def _get_neighbors(position: Position, cave: numpy.ndarray) -> list[Position]:
    neighbors: list[Position] = []
    for new_position in [(1, 0), (-1, 0), (0, 1), (0, -1)]:
        node_position = position[0] + new_position[0], position[1] + new_position[1]
        if (
            0 <= node_position[0] < cave.shape[0]
            and 0 <= node_position[1] < cave.shape[1]
        ):
            neighbors.append(node_position)
    return neighbors

def _a_star(start: Position, end: Position, cave: numpy.ndarray) -> int:
    """A тнР search. Implementation inspired by www.redblobgames.com."""
    frontier = PriorityQueue()
    frontier.put(start, 0)
    came_from: dict[Position, Optional[Position]] = {start: None}
    cost_so_far: dict[Position, float] = {start: 0.0}

    while not frontier.empty():
        current = frontier.get()
        if current == end:
            break
        for next_pos in _get_neighbors(current, cave):
            new_cost = cost_so_far.get(current) + cave[next_pos]
            if next_pos not in cost_so_far or new_cost < cost_so_far[next_pos]:
                cost_so_far[next_pos] = new_cost
                priority = new_cost + _heuristic(next_pos, end)
                frontier.put(next_pos, priority)
                came_from[next_pos] = current

    return int(cost_so_far.get(end, 0))

def _multiply_cave(cave: numpy.ndarray, k: int) -> numpy.ndarray:
    return numpy.block(
        [[(cave + i + j - 1) % 9 + 1 for i in range(k)] for j in range(k)]
    )

def find_path(filename: str) -> int:
    cave = _parse_input(filename)
    return _a_star((0, 0), (cave.shape[0] - 1, cave.shape[1] - 1), cave)

def find_path_bigger(filename: str, k: int) -> int:
    cave = _parse_input(filename)
    cave = _multiply_cave(cave, k)
    return _a_star((0, 0), (cave.shape[0] - 1, cave.shape[1] - 1), cave)

print(
    (
        find_path("15.txt"),
        find_path_bigger("15.txt", 5),
    )
)


from db import db

post_tags = db.Table('post_tags',
    db.Column('tag_id', db.Integer, db.ForeignKey('tags.id'), primary_key=True),
    db.Column('post_id', db.Integer, db.ForeignKey('posts.id'), primary_key=True)
)

class TagModel(db.Model):
    __tablename__ = 'tags'

    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(255), nullable=False, unique=True)

    posts = db.relationship('PostModel',secondary=post_tags, back_populates="tags")

    def json(self):
        return {
            "id":self.id,
            "name":self.name
        }


from flask import Flask, render_template, g, request, session, redirect, url_for
from flask_socketio import SocketIO
from database import get_db

app = Flask(__name__)

app.config['DEBUG'] = True
app.config['SECRET_KEY'] = 'secretkeyforme'
socketio = SocketIO(app)

@app.teardown_appcontext
def close_db(error):
    if hasattr(g,'postgres_db_cur'):
        g.postgres_db_cur.close()

    if hasattr(g, 'postgres_db_conn'):
        g.postgres_db_conn.close()

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/query')
def query():
    id = request.args.get('id')
    x = request.args.get('x')
    y = request.args.get('y')
    z = request.args.get('z')
    db = get_db()
    db.execute('insert into location(id, x, y, z) values (%s, %s, %s, %s);', (id, x, y, z,))
    return '<h1>{} {} {} {}</h1>'.format(id, x, y, z)

@app.route('/viewresult')
def viewresult():
    db = get_db()
    db.execute('select id, x, y, z from location')
    results = db.fetchall()
    return render_template('viewresult.html', result = results)

@app.route('/websocket')
def websocket():
    return render_template('websocket.html')

@app.route('/map')
def map():
    return render_template('map.html')

@socketio.on('message')
def receive_message(message):
    print('########: {}'.format(message))

@socketio.on('sending_event')
def receive_sending_event(message):
    print('ID: {}, X = {}, Y = {}, Z = {}'.format(message['id'],message['x'],message['y'],message['z']))
    print('stufF: {}'.format(message['imgbyte']))
    socketio.emit('render_page', message)

if __name__ == '__main__':
    socketio.run(app)


from typing import Type

from dlt.common.schema.schema import Schema
from dlt.common.configuration import with_config, known_sections
from dlt.common.configuration.accessors import config
from dlt.common.destination import DestinationCapabilitiesContext
from dlt.common.destination.reference import JobClientBase, DestinationClientConfiguration

from dlt.destinations.dummy.configuration import DummyClientConfiguration

@with_config(spec=DummyClientConfiguration, sections=(known_sections.DESTINATION, "dummy",))
def _configure(config: DummyClientConfiguration = config.value) -> DummyClientConfiguration:
    return config

def capabilities() -> DestinationCapabilitiesContext:
    config = _configure()
    caps = DestinationCapabilitiesContext()
    caps.preferred_loader_file_format = config.loader_file_format
    caps.supported_loader_file_formats = [config.loader_file_format]
    caps.preferred_staging_file_format = None
    caps.supported_staging_file_formats = []
    caps.max_identifier_length = 127
    caps.max_column_identifier_length = 127
    caps.max_query_length = 8 * 1024 * 1024
    caps.is_max_query_length_in_bytes = True
    caps.max_text_data_type_length = 65536
    caps.is_max_text_data_type_length_in_bytes = True
    caps.supports_ddl_transactions = False

    return caps

def client(schema: Schema, initial_config: DestinationClientConfiguration = config.value) -> JobClientBase:
    # import client when creating instance so capabilities and config specs can be accessed without dependencies installed
    from dlt.destinations.dummy.dummy import DummyClient

    return DummyClient(schema, _configure(initial_config))  # type: ignore

def spec() -> Type[DestinationClientConfiguration]:
    return DummyClientConfiguration


import json
import pygal
from pygal.style import RotateStyle
from countries_codes import get_country_code
#   Load the data into a list
filename = "population_data.json"
with open(filename) as f :
  pop_data = json.load(f)
#   Buidl a dictionary of population data
cc_populations = {}
#   Print the 2010 population for each country
for pop_dict in pop_data :
  if pop_dict["Year"] =="2010" :
    country_name = pop_dict["Country Name"]
    population = int(float(pop_dict["Value"]))
    code = get_country_code(country_name)
    if code :
      cc_populations[code] = population
    #   print(code + ": " + str(population))
    # else :
    #   print("Error - " + country_name)
    # #print(country_name + ": " + str(population))
cc_pops_1, cc_pops_2, cc_pops_3 = {}, {}, {}
for cc, pop in cc_populations.items() :
  if pop < 10000000 :
    cc_pops_1[cc] = pop
  elif pop < 1000000000 :
    cc_pops_2[cc] = pop
  else :
    cc_pops_3[cc] = pop
#   See how many countries are in each level.
print(len(cc_pops_1), len(cc_pops_2), len(cc_pops_3))
wm_style = RotateStyle("#336699")
wm = pygal.maps.world.World(style = wm_style)
wm.title = "World Population in 2010, by County"
wm.add("0-10m", cc_pops_1)
wm.add("10m-10n", cc_pops_2)
wm.add(">10b", cc_pops_3)
# wm.add("2010", cc_populations)
wm.render_to_file("world_population.svg")


# C:\Users\zhenqiang.huang\Desktop\searchprocess\pydata
def get_file_content(path, name):
    read = open(path + "/" + name, "r", encoding="utf-8")

    content = read.read()
    read.close()
    return content
def write_file(path,content):
    wrt = open(path, "w", encoding="utf-8")
    wrt.write(content)
    wrt.close()

if __name__ == '__main__':
    path = "C:/Users/zhenqiang.huang/Desktop/searchprocess/pydata"
    content = get_file_content(path, "source.txt")

    cntarr = content.split("\n")

    cntarr.sort()

    print(cntarr)

    # write_file(path+"/copy.txt",content+"cntarr")


''' Problem 4 exam
power spectrum of random data
SAGAR DAM'''

import numpy as np
import matplotlib.pyplot as plt

data = np.random.rand(1024)
n=len(data)
h=1
x=np.arange(0,n,1)

#plotting data points
plt.plot(x,data,'ko',markersize=1,label='plotting the random points')
plt.xlabel('x')
plt.ylabel('noise value')
plt.legend()
plt.grid()
plt.title('scatter plot of random points')
plt.show()

#histogram of distribution:
plt.hist(data, density='true', color='y',edgecolor='k',bins=np.arange(0,1+0.05,0.05))
plt.title('histogram of random data points')
plt.show()

#Doing DFT of given data (using numpy package)
k=np.fft.fftshift(np.fft.fftfreq(n,h))
print("k_max= ",np.amax(k))
print("k_min= ",np.amin(k))
gk=abs(np.fft.fftshift(np.fft.fft(data,norm='ortho')))
plt.plot(k,gk,'r',label='DFT')
plt.suptitle('Plotting the DFT of given data')
plt.xlabel('w')
plt.ylabel('g(w)')
plt.legend()
plt.grid()
plt.show()

#power spectrum;  p~A**2
plt.plot(k,gk**2)
plt.xlabel('w')
plt.ylabel('Intensity(w)')
plt.title('power spectrum of random numbers')
plt.legend()
plt.show()

#Binning the power spectrum
b=5
no=int(n/b)
ko=np.fft.fftshift(np.fft.fftfreq(no,h))
hk=np.ones(no*b,dtype=float)
hk=hk.reshape(b,no)
s=np.zeros(no,dtype=float)
h=data[0:no*b].reshape(b,no)
for i in range(5):
    hk[i]=np.fft.fftshift(abs(np.fft.fft(h[i],norm='ortho')))
    s=s+((hk[i])**2)
    plt.plot(ko,s,label='binned power spectrum')
plt.legend()
plt.grid()
plt.title('binned power spectrum')
plt.show()


#! /usr/bin/env python

# This file is part of Paparazzi.
#
# Paparazzi is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# Paparazzi is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Paparazzi; see the file COPYING.  If not, write to
# the Free Software Foundation, 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
#

import sys
import os
from optparse import OptionParser

import calibration_utils

def main():
    usage = "usage: %prog [options] log_filename.data" + "\n" + "Run %prog --help to list the options."
    parser = OptionParser(usage)
    parser.add_option("-i", "--id", dest="ac_id",
                      action="store",
                      help="aircraft id to use")
    parser.add_option("-p", "--plot",
                      help="Show sensor plots",
                      action="store_true", dest="plot")
    parser.add_option("-s", "--start", dest="start",
                      action="store",
                      type=int, default=0,
                      help="start time in seconds")
    parser.add_option("-e", "--end", dest="end",
                      action="store",
                      type=int, default=36000,
                      help="end time in seconds")
    parser.add_option("-v", "--verbose",
                      action="store_true", dest="verbose")
    (options, args) = parser.parse_args()

    if len(args) != 1:
        parser.error("incorrect number of arguments")
    else:
        if os.path.isfile(args[0]):
            filename = args[0]
        else:
            print(args[0] + " not found")
            sys.exit(1)

    ac_ids = calibration_utils.get_ids_in_log(filename)
    if options.ac_id is None:
        if len(ac_ids) == 1:
            options.ac_id = ac_ids[0]
        else:
            parser.error("More than one aircraft id found in log file. Specify the id to use.")
    if options.verbose:
        print("Using aircraft id "+options.ac_id)

    if not filename.endswith(".data"):
        parser.error("Please specify a *.data log file")
    if options.verbose:
        print("reading file "+filename+" for aircraft "+options.ac_id+" and scaled sensors")

    #Moved these checks to the command line parser above
    #
    #if options.start is None:
    #    options.start = 0
    #if options.end is None:
    #    options.end = 36000

    # read scaled sensor measurements from log file
    # TBD: Eventually populate the sensor attributes/values with data found in the messages.xml file
    sensor_names  = [ "ACCEL", "GYRO", "MAG" ]
    sensor_attrs  = [ [0.0009766, "m/s2", "ax", "ay", "az"], [0.0139882, "deg/s", "gp", "gq", "gr"], [0.0004883, "unit", "mx", "my", "mz"] ]

    for sensor_name in sensor_names:
        measurements = calibration_utils.read_log_scaled(options.ac_id, filename, sensor_name, options.start, options.end)
        if len(measurements) > 0:
            if options.verbose:
                print("found "+str(len(measurements))+" records")
            calibration_utils.print_imu_scaled(sensor_name, measurements, sensor_attrs[sensor_names.index(sensor_name)])
            if options.plot:
                calibration_utils.plot_imu_scaled(sensor_name, measurements, sensor_attrs[sensor_names.index(sensor_name)])
                calibration_utils.plot_imu_scaled_fft(sensor_name, measurements, sensor_attrs[sensor_names.index(sensor_name)])
        else:
            print("Warning: found zero IMU_"+sensor_name+"_SCALED measurements for aircraft with id "+options.ac_id+" in log file!")
            #sys.exit(1)

    print("")

    # coefficient = calibration_utils.estimate_mag_current_relation(measurements)

    # print("")
    # print("<define name= \"MAG_X_CURRENT_COEF\" value=\""+str(coefficient[0])+"\"/>")
    # print("<define name= \"MAG_Y_CURRENT_COEF\" value=\""+str(coefficient[1])+"\"/>")
    # print("<define name= \"MAG_Z_CURRENT_COEF\" value=\""+str(coefficient[2])+"\"/>")

if __name__ == "__main__":
    main()


class Restaurant:
    number_served = 0
    def __init__(self,name,type):
        self.restaurant_name = name
        self.cuisine_type = type
    def describ1e_restaurant(self):
        print("저희 레스토랑 명칭은 %s이고 %s 전문점입니다." %(self.restaurant_name,self.cuisine_type))
    def open_restaurant(self):
        print("저희 %s 레스토랑 오픈했습니다. 어서오세요." %(self.restaurant_name))
    def set_number_served(self,number):
        self.number_served1 = number
        print("오늘 방문한 고객은 %s명 입니다." % (self.number_served1))
    def increment_number_served(self,number):
        self.number_served = number + self.number_served
        print("오늘 총방문수는 %s명 입니다." % (self.number_served))

restaurant = Restaurant("빈센트","스테이크")
art = Restaurant("빈센트","스테이크")
art.describe_restaurant()
art.open_restaurant()

restaurant.set_number_served(0)
restaurant.increment_number_served(20)
restaurant.increment_number_served(20)
restaurant.increment_number_served(20)
restaurant.increment_number_served(20)

f = open("D:\python_workspace\Jumptopy\Codding_Test\고객서빙현황로그.txt",'r')
line = f.readlines()
f.close()
f = open("D:\python_workspace\Jumptopy\Codding_Test\고객서빙현황로그.txt",'a')
a=line[-1]
b=int(a)+restaurant.number_served
f.write(str(b)+'\n')
f.close()


import sys
import os

# add path that contains the dominoes package
mainPath = os.path.dirname(os.path.abspath(__file__)) + "/.."
sys.path.append(mainPath)

# standard imports
from copy import copy
import argparse
from tqdm import tqdm
import numpy as np
import scipy as sp
import matplotlib.pyplot as plt
import matplotlib as mpl
import torch
import torch.cuda as torchCuda

# dominoes package
from dominoes import fileManagement as fm
from dominoes import datasets
from dominoes import training
from dominoes import transformers
from dominoes.utils import loadSavedExperiment
from dominoes import utils

device = 'cuda' if torchCuda.is_available() else 'cpu'

# general variables for experiment
POINTER_METHODS = ['PointerStandard', 'PointerDot', 'PointerDotLean', 'PointerDotNoLN', 'PointerAttention', 'PointerTransformer']

# path strings
netPath = fm.netPath()
resPath = fm.resPath()
prmsPath = fm.prmPath()
figsPath = fm.figsPath()

for path in (resPath, prmsPath, figsPath, netPath):
    if not(path.exists()):
        path.mkdir()

# method for returning the name of the saved network parameters (different save for each possible opponent)
def getFileName(extra=None):
    baseName = "pointerArchitectureComparison"
    if extra is not None:
        baseName = baseName + f"_{extra}"
    return baseName

def handleArguments():
    parser = argparse.ArgumentParser(description='Run pointer demonstration.')
    parser.add_argument('-hd','--highest-dominoe', type=int, default=9, help='the highest dominoe in the board')
    parser.add_argument('--train-fraction', type=float, default=0.8, help='the fraction of dominoes in the set to train with')
    parser.add_argument('-hs','--hand-size', type=int, default=8, help='tokens per sequence')
    parser.add_argument('-bs','--batch-size',type=int, default=512, help='number of sequences per batch')
    parser.add_argument('-ne','--train-epochs',type=int, default=4000, help='the number of training epochs')
    parser.add_argument('-te','--test-epochs',type=int, default=100, help='the number of testing epochs')
    parser.add_argument('--gamma', type=float, default=1.0, help='discounting factor')
    parser.add_argument('-nr','--num-runs', type=int, default=8, help='how many networks to train of each type')

    parser.add_argument('--embedding-dim', type=int, default=48, help='the dimensions of the embedding')
    parser.add_argument('--heads', type=int, default=1, help='the number of heads in transformer layers')
    parser.add_argument('--encoding-layers', type=int, default=1, help='the number of stacked transformers in the encoder')
    parser.add_argument('--expansion', type=int, default=4, help='the expansion of the feedforward layer in the transformer of the encoder')
    parser.add_argument('--train-temperature',type=float, default=5.0, help='temperature for training')

    parser.add_argument('--justplot', default=False, action='store_true', help='if used, will only plot the saved results (results have to already have been run and saved)')
    parser.add_argument('--nosave', default=False, action='store_true')
    parser.add_argument('--printargs', default=False, action='store_true')

    args = parser.parse_args()

    assert args.train_fraction > 0 and args.train_fraction <= 1, "train fraction must be greater than 0 and less than or equal to 1"
    assert args.gamma > 0 and args.gamma <= 1, "gamma must be greater than 0 or less than or equal to 1"

    return args

def trainTestDominoes(baseDominoes, trainFraction):
    # figure out which dominoes need to be represented in both ways (ones with different values)
    doubleDominoes = baseDominoes[:,0] == baseDominoes[:,1]
    nonDoubleReverse = baseDominoes[~doubleDominoes][:,[1,0]] # represent each dominoe in both orders

    # list of full set of dominoe representations and value of each
    fullDominoes = np.concatenate((baseDominoes, nonDoubleReverse), axis=0)

    # subset dominoes for training
    trainNumber = int(len(fullDominoes)*trainFraction)
    trainIndex = np.sort(np.random.permutation(len(fullDominoes))[:trainNumber]) # keep random fraction (but sort in same way)
    trainDominoes = fullDominoes[trainIndex]

    return fullDominoes, trainDominoes, trainIndex

def trainTestModel():
    # get values from the argument parser
    highestDominoe = args.highest_dominoe
    baseDominoes = utils.listDominoes(highestDominoe)

    # other batch parameters
    ignoreIndex = -1 # this is only used when doing uneven batch sizes, which I'm not in this experiment
    handSize = args.hand_size
    batchSize = args.batch_size
    maxOutput = copy(handSize)

    # network parameters
    input_dim = 2*(highestDominoe+1)
    embedding_dim = args.embedding_dim
    heads = args.heads
    encoding_layers = args.encoding_layers
    expansion = args.expansion

    # policy parameters
    with_thompson = True
    temperature = args.train_temperature

    # train parameters
    trainEpochs = args.train_epochs
    testEpochs = args.test_epochs
    numRuns = args.num_runs

    # create gamma transform matrix
    gamma = args.gamma
    exponent = torch.arange(maxOutput).view(-1,1) - torch.arange(maxOutput).view(1,-1)
    gamma_transform = (gamma ** exponent * (exponent >= 0)).to(device)

    # batch inputs for reinforcement learning training
    batch_inputs = {'null_token':False,
                    'available_token':False,
                    'ignore_index':ignoreIndex,
                    'return_full':True,
                    'return_target':False,
                   }

    numNets = len(POINTER_METHODS)

    # Train network
    print("Training networks...")
    trainReward = torch.zeros((trainEpochs, numNets, numRuns))
    testReward = torch.zeros((testEpochs, numNets, numRuns))
    trainRewardByPos = torch.zeros((trainEpochs, maxOutput, numNets, numRuns))
    testRewardByPos = torch.zeros((testEpochs, maxOutput, numNets, numRuns))
    trainScoreByPos = torch.zeros((trainEpochs, maxOutput, numNets, numRuns))
    testScoreByPos = torch.zeros((testEpochs, maxOutput, numNets, numRuns))
    for run in range(numRuns):
        # reset train set of dominoes
        fullDominoes, trainDominoes, trainIndex = trainTestDominoes(baseDominoes, args.train_fraction)

        # create pointer networks with different pointer methods
        nets = [transformers.PointerNetwork(input_dim, embedding_dim, temperature=temperature, pointer_method=POINTER_METHOD,
                                             thompson=with_thompson, encoding_layers=encoding_layers, heads=heads, kqnorm=True,
                                             decoder_method='transformer', expansion=expansion)
                 for POINTER_METHOD in POINTER_METHODS]
        nets = [net.to(device) for net in nets]

        # Create an optimizer, Adam with weight decay is pretty good
        optimizers = [torch.optim.Adam(net.parameters(), lr=1e-3, weight_decay=1e-5) for net in nets]

        print(f"Doing training run {run+1}/{numRuns}...")
        for epoch in tqdm(range(trainEpochs)):
            batch = datasets.generateBatch(highestDominoe, trainDominoes, batchSize, handSize, **batch_inputs)

            # unpack batch tuple
            input, _, _, _, _, selection, _ = batch
            input = input.to(device)

            # zero gradients, get output of network
            for opt in optimizers: opt.zero_grad()
            log_scores, choices = map(list, zip(*[net(input, max_output=maxOutput) for net in nets]))

            # log-probability for each chosen dominoe
            logprob_policy = [torch.gather(score, 2, choice.unsqueeze(2)).squeeze(2) for score, choice in zip(log_scores, choices)]

            # measure reward
            rewards = [training.measureReward_sortDescend(trainDominoes[selection], choice) for choice in choices]
            G = [torch.matmul(reward, gamma_transform) for reward in rewards]

            # measure J
            J = [-torch.sum(logpol * g) for logpol, g in zip(logprob_policy, G)]
            for j in J: j.backward()

            # update networks
            for opt in optimizers: opt.step()

            # save training data
            with torch.no_grad():
                for i in range(numNets):
                    trainReward[epoch, i, run] = torch.mean(torch.sum(rewards[i], dim=1)).detach()
                    trainRewardByPos[epoch, :, i, run] = torch.mean(rewards[i], dim=0).detach()

                    # Measure the models confidence -- ignoring the effect of temperature
                    pretemp_score = torch.softmax(log_scores[i] * nets[i].temperature, dim=2)
                    pretemp_policy = torch.gather(pretemp_score, 2, choices[i].unsqueeze(2)).squeeze(2)
                    trainScoreByPos[epoch, :, i, run] = torch.mean(pretemp_policy, dim=0).detach()

        with torch.no_grad():
            # always return temperature to 1 and thompson to False for testing networks
            for net in nets:
                net.setTemperature(1.0)
                net.setThompson(False)

            print('Testing network...')
            for epoch in tqdm(range(testEpochs)):
                batch = datasets.generateBatch(highestDominoe, fullDominoes, batchSize, handSize, **batch_inputs)

                # unpack batch tuple
                input, _, _, _, _, selection, _ = batch

                # move to correct device
                input = input.to(device)

                # get output of networks
                log_scores, choices = map(list, zip(*[net(input, max_output=maxOutput) for net in nets]))

                # log-probability for each chosen dominoe
                logprob_policy = [torch.gather(score, 2, choice.unsqueeze(2)).squeeze(2) for score, choice in zip(log_scores, choices)]

                # measure reward
                rewards = [training.measureReward_sortDescend(fullDominoes[selection], choice) for choice in choices]

                # save training data
                for i in range(numNets):
                    testReward[epoch, i, run] = torch.mean(torch.sum(rewards[i], dim=1))
                    testRewardByPos[epoch, :, i, run] = torch.mean(rewards[i], dim=0)

                    # Measure the models confidence -- ignoring the effect of temperature
                    pretemp_score = torch.softmax(log_scores[i] * nets[i].temperature, dim=2)
                    pretemp_policy = torch.gather(pretemp_score, 2, choices[i].unsqueeze(2)).squeeze(2)
                    testScoreByPos[epoch, :, i, run] = torch.mean(pretemp_policy, dim=0).detach()

    results = {
        'trainReward': trainReward,
        'testReward': testReward,
        'trainRewardByPos': trainRewardByPos,
        'testRewardByPos': testRewardByPos,
        'trainScoreByPos': trainScoreByPos,
        'testScoreByPos': testScoreByPos,
    }

    return results, nets

@torch.no_grad()
def eigenAnalyses(nets, args):
    # make sure everything is on the same device
    nets = [net.to(device) for net in nets]

    # get a "normal" batch
    highestDominoe = args.highest_dominoe
    listDominoes = utils.listDominoes(highestDominoe)

    # do subselection for training
    doubleDominoes = listDominoes[:,0] == listDominoes[:,1]
    nonDoubleReverse = listDominoes[~doubleDominoes][:,[1,0]] # represent each dominoe in both orders

    # list of full set of dominoe representations and value of each
    listDominoes = np.concatenate((listDominoes, nonDoubleReverse), axis=0)
    dominoeValue = np.sum(listDominoes, axis=1)

    # training inputs
    numDominoes = len(listDominoes)
    dominoeValue = np.sum(listDominoes, axis=1)
    batchSize = args.batch_size * 2 # lots of data!
    handSize = args.hand_size
    batch_inputs = dict(
        null_token=False,
        available_token=False,
        ignore_index=- 1,
        return_full=True,
        return_target=False
    )

    selection = np.array([])
    while len(np.unique(selection))!=numDominoes:
        batch = datasets.generateBatch(highestDominoe, listDominoes, batchSize, handSize, **batch_inputs)

        # unpack batch tuple
        input, _, _, _, _, selection, available = batch
        input = input.to(device)

    # pre-forward
    batch_size, tokens, _ = input.size()
    mask = torch.ones((batch_size, tokens), dtype=input.dtype).to(device)

    # encoding
    embedded = [net.embedding(input) for net in nets]
    encoded = [net.encoding(embed) for net, embed in zip(nets, embedded)]

    # translate to (token x embedding)
    encoded = [encode.view(-1, encode.size(2)).T for encode in encoded]

    # get covariance
    cov = [torch.cov(encode) for encode in encoded]

    # get eigenvalues
    eigvals, eigvecs = map(list, zip(*[torch.linalg.eigh(c) for c in cov]))
    eigvals = [eigval.cpu().numpy() for eigval in eigvals]
    eigvecs = [eigvec.cpu().numpy() for eigvec in eigvecs]

    # sort highest to lowest
    idx_sort = [np.argsort(-eigval) for eigval in eigvals]
    eigvals = [eigval[isort] for eigval, isort in zip(eigvals, idx_sort)]
    eigvecs = [eigvec[:,isort] for eigvec, isort in zip(eigvecs, idx_sort)]

    # negative values are numerical errors
    eigvals = [np.maximum(eigval, 0) for eigval in eigvals]

    return eigvals, eigvecs

def plotResults(results, args, eigvals):
    numNets = results['trainReward'].shape[2]
    numPos = results['testScoreByPos'].shape[1]
    n_string = f" (N={numNets})"
    cmap = mpl.colormaps['tab10']
    trainInspectFrom = [200, 300]
    trainInspect = slice(trainInspectFrom[0], trainInspectFrom[1])

    fig, ax = plt.subplots(1,2,figsize=(7,3.5), width_ratios=[2, 1], layout='constrained')
    for idx, name in enumerate(POINTER_METHODS):
        ax[0].plot(range(args.train_epochs), torch.mean(results['trainReward'][:,idx],dim=1), color=cmap(idx), lw=1.2, label=name)
    ax[0].set_xlabel('Training Epoch')
    ax[0].set_ylabel('Mean Reward'+n_string)
    ax[0].set_title('Training Performance')
    ax[0].set_xlim(-50, 1000)
    ax[0].set_ylim(None, 8.005)
    ax[0].legend(loc='lower right', fontsize=9)
    yMin0, yMax0 = ax[0].get_ylim()

    xOffset = [-0.2, 0.2]
    get_x = lambda idx: [xOffset[0]+idx, xOffset[1]+idx]
    for idx, name in enumerate(POINTER_METHODS):
        mnTestReward = torch.nanmean(results['testReward'][:,idx], dim=0)
        ax[1].plot(get_x(idx), [mnTestReward.mean(), mnTestReward.mean()], color=cmap(idx), lw=4, label=name)
        for mtr in mnTestReward:
            ax[1].plot(get_x(idx), [mtr, mtr], color=cmap(idx), lw=1.5)
        ax[1].plot([idx,idx], [mnTestReward.min(), mnTestReward.max()], color=cmap(idx), lw=1.5)
    ax[1].set_xticks(range(len(POINTER_METHODS)))
    ax[1].set_xticklabels([pmethod[7:] for pmethod in POINTER_METHODS], rotation=45, ha='right', fontsize=8)
    ax[1].set_ylabel('Reward'+n_string)
    ax[1].set_title('Testing Performance')
    ax[1].set_xlim(-1, len(POINTER_METHODS))
    ax[1].set_ylim(7.85, 8.005)
    ax[1].set_yticks([7.85, 7.9, 7.95, 8])

    width = trainInspectFrom[1]-trainInspectFrom[0]
    height = yMax0 - yMin0
    rect = mpl.patches.Rectangle([trainInspectFrom[0], yMin0], width, height, facecolor='k', edgecolor='none', alpha=0.2)
    ax[0].add_patch(rect)

    if not(args.nosave):
        plt.savefig(str(figsPath/getFileName()))

    plt.show()

    # now show confidence by position figure
    fig, ax = plt.subplots(1, 2, figsize=(7, 3.5), layout='constrained')
    for idx, name in enumerate(POINTER_METHODS):
        mnTrainScore = torch.mean(results['trainScoreByPos'][trainInspect, :, idx], dim=(0,2))
        ax[0].plot(range(numPos), mnTrainScore, color=cmap(idx), lw=1.2, label=name)
    for idx, name in enumerate(POINTER_METHODS):
        mnTrainScore = torch.mean(results['trainScoreByPos'][trainInspect, :, idx], dim=(0,2))
        ax[0].scatter(range(numPos), mnTrainScore, color=cmap(idx), marker='o', s=24, edgecolor='none')
    ax[0].set_ylim(None, 1)
    ax[0].set_xlabel('Output Position')
    ax[0].set_ylabel('Mean Score'+n_string)
    ax[0].set_title(f"Train Confidence (in grey patch)")
    yMin2, yMax2 = ax[0].get_ylim()

    for idx, name in enumerate(POINTER_METHODS):
        mnTestScore = torch.mean(results['testScoreByPos'][:, :, idx], dim=(0,2))
        ax[1].plot(range(numPos), mnTestScore, color=cmap(idx), lw=1.2, marker='o', markersize=np.sqrt(24), label=name)
    ax[1].set_xlabel('Output Position')
    ax[1].set_ylabel('Mean Score'+n_string)
    ax[1].set_title('Test Confidence')
    ax[1].legend(loc='lower right', fontsize=9)
    yMin3, yMax3 = ax[1].get_ylim()

    new_ymin = min(yMin2, yMin3)
    ax[0].set_ylim(new_ymin, 1.005)
    ax[1].set_ylim(new_ymin, 1.005)

    if not(args.nosave):
        plt.savefig(str(figsPath/getFileName('confidence')))

    plt.show()

    # Show eigenvalue analysis
    fig, ax = plt.subplots(1, 2, figsize=(7, 3.5), width_ratios=[2,1], layout='constrained')
    for idx, (name, eigval) in enumerate(zip(POINTER_METHODS, eigvals)):
        ax[0].plot(range(len(eigval)), eigval/np.sum(eigval), color=cmap(idx), lw=1.2, label=name)
    ax[0].set_xlim(-1, len(eigval))
    ax[0].set_xlabel('Dimension')
    ax[0].set_ylabel('Fraction of Variance')
    ax[0].set_title('Eigenspectrum')
    ax[0].set_yscale('log')
    ax[0].legend(loc='lower left', fontsize=9)

    for idx, (name, eigval) in enumerate(zip(POINTER_METHODS, eigvals)):
        ax[1].plot(idx, sp.stats.entropy(eigval), color=cmap(idx), marker='o', linestyle='none', label=name)
    ax[1].set_ylabel('Entropy of Eigenspectrum')
    ax[1].set_title('Dimensionality')
    ax[1].set_xticks(range(len(POINTER_METHODS)))
    ax[1].set_xticklabels([pmethod[7:] for pmethod in POINTER_METHODS], rotation=45, ha='right', fontsize=8)
    ax[1].set_xlim(-1, len(POINTER_METHODS))
    _, yMax = ax[1].get_ylim()
    ax[1].set_ylim(0, 1.65)
    ax[1].set_yticks(np.linspace(0, 1.6, 5))

    if not(args.nosave):
        plt.savefig(str(figsPath/getFileName('eigenvalues')))

    plt.show()

if __name__=='__main__':
    args = handleArguments()
    show_results = True

    if args.printargs:
        _, args = loadSavedExperiment(prmsPath, resPath, getFileName(), args=args)
        for key, val in vars(args).items():
            print(f"{key}={val}")
        show_results = False

    elif not(args.justplot):
        results, nets = trainTestModel()
        nets = [net.to('cpu') for net in nets]

        # save results if requested
        if not(args.nosave):
            # Save agent parameters
            for net, method in zip(nets, POINTER_METHODS):
                save_name = f"{method}.pt"
                torch.save(net, netPath / getFileName(extra=save_name))
            np.save(prmsPath / getFileName(), vars(args))
            np.save(resPath / getFileName(), results)

    else:
        results, args = loadSavedExperiment(prmsPath, resPath, getFileName(), args=args)
        nets = [torch.load(netPath / getFileName(extra=f"{method}.pt")) for method in POINTER_METHODS]

    if show_results:
        eigval, eigvec = eigenAnalyses(nets, args)
        plotResults(results, args, eigval)


import requests
import json

import pandas as pd
from ..source import Source, URIs
from sources.pycountry_util import find_country

from .secret import PASSWORD as password

class Banktrack(Source):
    """
    Data from BankTrack.org, a nonprofit that tracks questionable bank financing.
    The BankTrack.org project also provides unique ID's for each of the ~220 banks
    in its dataset, which this project uses.
    Because of this, BankTrack data must be imported before other data sources.
    """
    def __init__(self, bankreg, name, tag, update_date,
                 banktrack_link, country, website, description=''):

        # in the case of banktrack, a source_id is supplied by
        # the original data provider
        self.source_id = tag

        self.name = name.rstrip().lstrip()
        self.description = description
        self.update_date = update_date
        self.banktrack_link = banktrack_link
        self.countries = set([find_country(country)[1]])
        self.website = website

        super(Banktrack, self).__init__(bankreg=bankreg,
                                        name=name,
                                        countries=set([find_country(country)[1]]))

    @property
    def tag(self):
        '''overwrites parent property. Must be lowercased and stripped to match pipeline expectations'''
        return self.source_id.lower().rstrip().lstrip()

    @classmethod
    def load_and_create(cls, bankreg, load_from_api=False):

        # load from api or from local disk.
        # this is here because we don't have permission to publish one column of the data in this table
        # and definitely don't have permission for opening up BankTrack's api.
        df = None
        if not load_from_api:
            df = pd.read_csv('./sources/banktrack/bankprofiles.csv')
        else:
            r = requests.post(URIs.BANK_TRACK.value, data={'pass': password})
            res = json.loads(r.text)
            df = pd.DataFrame(res['bankprofiles'])
            df = df.drop(columns=['general_comment'])
            df.to_csv('bankprofiles.csv')

        for i, row in df.iterrows():
            bank = Banktrack(bankreg=bankreg,
                             name=row.title,
                             tag=row.tag,
                             description=row.general_comment if 'general_comment' in row.values else '',
                             update_date=row.updated_at,
                             banktrack_link=row.link,
                             country=row.country,
                             website=row.website)
            bankreg.create_or_update_bank(source=bank)
        return bankreg


"""
If we list all the natural numbers below 10 that are multiples of 3 or 5,
we get 3, 5, 6 and 9. The sum of these multiples is 23.

Find the sum of all the multiples of 3 or 5 below 1000.
"""

def solve_problem():

    sum = 0

    for x in range(1 , 1000):
        if x % 3 == 0:
            sum += x
        elif x % 5 == 0:
            sum += x

    return sum

if __name__ == '__main__':
    print(solve_problem())


# -*- coding: utf-8 -*-
from __future__ import print_function
__author__ = 'factory'
__version__ = "1.0.0"

import os, sys, re, time

import subprocess
import numpy as np
from tools import *
from geodata import copydeflate, SaveRasterBands

init_image = None
final_image = None
input_folder = r'e:\rks\rucode\new4\reproject' # Путь к исходным растровым данным
output_folder = r'e:\rks\rucode\new4\composit' # Путь для сохранения итоговых композитов
bands_order = [1, 2, 3, 4, 1, 2, 3, 4] # Порядок каналов; для обработки Planet использовать [3,2,1,4,3,2,1,4]
images_order = ['i','i','i','i','f','f','f','f'] # Порядок изображений для отбора каналов (4 с первого снимка, 4 с последнего)
path2xls = r'e:\rks\rucode\new4\composit\composit.xls' # Путь к таблице с названиями файлов для коррегистрации, указанных соответственно в колонках 'old' и 'new'; в колонке 'composit' должно быть указано уникальное имя конечного файла (расширение можно не указывать)

def Usage():
    print(
       """
Tool for KANOPUS L2 APOI change detection (composit builder).
init_image_file final_image_file output_folder
[-io input images order][-bo input bands order]
init_image_file - init image file (i)
final_image_file - final image file (f)
output_folder - output folder
[-io] - input images order ('f,i,f' by default)
[-bo] - input bands order ('1,1,2' by default)
      """)
    sys.exit(1)

try:
    from pci.pansharp import *
    from pci.fexport import *
except:
    print('')
    print('PCI GEOMATICA not available.')
    print('')
    sys.exit(1)

try:
    import gdal, ogr, osr, gdalconst
except:
    print('')
    print('GDAL lib not available.')
    print('')
    sys.exit(1)

def runcmd(cmd):
    p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE)
    while True:
        out = p.stderr.read(1)
        if out == '' and p.poll() != None:
            break
        if out != '':
            sys.stdout.write(out)
            sys.stdout.flush()

def GeoTiffSetNodata(fname, value_nodata):
    ds = gdal.Open ( fname, gdalconst.GA_Update )
    if ds is not None:
        band = ds.GetRasterBand(1)
        band.SetNoDataValue(value_nodata)
        band.FlushCache()
        band = None
        ds = None

def BuildActiveAreaMask(images, bands):
        mask=np.ones((images[0].RasterYSize, images[0].RasterXSize), dtype=np.int8)
        for band in bands:
            band_data=images[0].GetRasterBand(band).ReadAsArray().astype(np.uint16)
            mask[band_data==0]=0
            band_data=images[1].GetRasterBand(band).ReadAsArray().astype(np.uint16)
            mask[band_data==0]=0
        return mask

def ImageProperties(ifile):
    ds = gdal.Open ( ifile, gdalconst.GA_ReadOnly )
    width = ds.RasterXSize
    height = ds.RasterYSize
    gt = ds.GetGeoTransform()
    minx = gt[0]
    miny = gt[3] + width*gt[4] + height*gt[5]
    maxx = gt[0] + width*gt[1] + height*gt[2]
    maxy = gt[3]
    template = 'POLYGON ((%(minx)f %(miny)f, %(minx)f %(maxy)f, %(maxx)f %(maxy)f, %(maxx)f %(miny)f, %(minx)f %(miny)f))'
    r1 = {'minx': minx, 'miny': miny, 'maxx':maxx, 'maxy':maxy}
    wkt = template % r1
    srs=osr.SpatialReference()
    srs.ImportFromWkt(ds.GetProjectionRef())
    ds = None
    return wkt, int(srs.GetAttrValue("AUTHORITY", 1)), srs, gt

def pci_auto_coregistration(i_image, reference_image, geo_correct_image, in_match_chan = [1], ref_match_chan = [1], bxpxsz="2.1", polyorder=[1], reject = [5, 1, 1], tresh=[0.88]):
    try:
        from pci.link import link
        from pci.autogcp2 import autogcp2
        from pci.gcprefn import gcprefn
        from pci.polymodel import polymodel
        from pci.ortho2 import ortho2
        ingest_link_file = i_image.split('.')[0]+'.pix'

        if os.path.exists(ingest_link_file):
            os.remove(ingest_link_file)

        link( i_image, ingest_link_file, [] )
        increase_radius = 0
        num_gcps = [0]

        while ((int(num_gcps[0]) > 6) == True) !=  ((increase_radius < 100) == True):
            search_radius = [15+increase_radius]
            smplsrc="GRID:256"
            algo="FFTP"
            gcp_seg = autogcp2( ingest_link_file, in_match_chan, u"", [], [], reference_image, ref_match_chan,
                                u"", [], [], u"", u"", [], smplsrc, algo, search_radius, u"", tresh, u"", num_gcps )
            increase_radius+=15
            tresh=[tresh[0]-0.01]

            if increase_radius>60:
                break

        reject = reject
        refined_gcp_seg = []
        out_stats = []
        gcprefn( ingest_link_file, [gcp_seg[-1]], [], [], "POLY", [], reject, "NO", refined_gcp_seg, out_stats )
        polymodel( ingest_link_file, [refined_gcp_seg[0]], polyorder, [] )
        ortho2(ingest_link_file, [], [], [], "", geo_correct_image, "TIF", "", [], "", "", "",
                "", [], "", "", bxpxsz, "", "", [], [], "", "", [], "", [], "")

        if os.path.exists(ingest_link_file):
            os.remove(ingest_link_file)

        return out_stats

    except:
        return []

def copydeflate(path_in, path_out, bigtiff = False, tiled = False):
    ds_in = gdal.Open(path_in)
    if ds_in is None:
        print('Cannot open file {}'.format(path_in))
        return 1
    dir_out = os.path.split(path_out)[0]
    if not os.path.exists(dir_out):
        os.makedirs(dir_out)
    options = ['COMPRESS=DEFLATE', 'PREDICTOR=2', 'ZLEVEL=9']
    if bigtiff:
        options.append('BIGTIFF=YES')
    if tiled:
        options.append('TILED=YES')
    driver = gdal.GetDriverByName('GTiff')
    ds_out = driver.CreateCopy(path_out, ds_in, options=options)
    ds_out = None
    print('File written {}'.format(path_out))
    return 0

# Import data from xls
def xls_to_dict(path2xls, sheetnum=0):
    rb = xlrd.open_workbook(path2xls)
    sheet = rb.sheet_by_index(sheetnum)
    keys = sheet.row_values(0)[1:]
    xls_dict = OrderedDict()
    for rownum in range(1, sheet.nrows):
        rowdata = OrderedDict()
        row = sheet.row_values(rownum)
        for key, val in zip(keys, row[1:]):
            rowdata[key] = val
        xls_dict[row[0]] = rowdata
    return xls_dict

# Searches filenames according to template and returns a list of full paths to them
def folder_paths(path, files = False, extension = None, id_max=10000):
    if os.path.exists(path):
        if not os.path.isdir(path):
            path = os.path.split(path)[0]
        path_list = [path]
        path_list = [path_list]
    else:
        print('Path does not exist: {}'.format(path))
        return None
    id = 0
    export_files = []
    while id < len(path_list) < id_max:
        for id_fold, folder in enumerate(path_list[id]):
            fold_, file_ = fold_finder(folder)
            if fold_ != []:
                path_list.append(fold_)
            if extension is None:
                export_files.extend(file_)
            else:
                for f in file_:
                    if f.endswith('.{}'.format(extension)):
                        export_files.append(f)
        id += 1
    if len(path_list) > id_max:
        raise Exception('Number of folder exceeds maximum = {}'.format(id_max))
    if files:
        return export_files
    export_folders = unite_multilist(path_list)
    return export_folders, export_files

# Returns a list of two lists: 0) all folders in the 'path' directory, 1) all files in it
def fold_finder(path):
    dir_ = os.listdir(path)
    fold_ = []
    file_ = []
    for name in dir_:
        full = path + '\\' + name
        if os.path.isfile(full):
            file_.append(full)
        else:
            fold_.append(full)
    return [fold_, file_]

# Unite all lists in a list of lists into a new list
def unite_multilist(list_of_lists):
    if len(list_of_lists) == 0:
        return []
    full_list = copy(list_of_lists[0])
    for list_ in list_of_lists[1:]:
        if isinstance(list_, list):
            full_list.extend(list_)
        else:
            print('Error: object is not a list: {}'.format(list_))
    return full_list

def scroll(obj, print_type=True, header=None):
    if header is not None:
        print(header)
    elif print_type:
        print('Object of {}:'.format(type(obj)))
    if hasattr(obj, '__iter__'):
        if len(obj) == 0:
            print('  <empty>')
        elif isinstance(obj, (dict, OrderedDict)):
            for val in obj:
                print('  {}: {}'.format(val, obj[val]))
        else:
            for val in obj:
                print('  {}'.format(val))
    else:
        print('  {}'.format(obj))

xls_data = xls_to_dict(path2xls)

test = xls_data

init_image_list = []
final_image_list = []
composit_image_list = []
channels_list = []

for i in xls_data.keys():
    test = xls_data[i]
    # scroll(test)
    init_test = test['old']#.replace('.PMS','.MS')
    final_test = test['new']#.replace('.PMS','.MS')
    for path in folder_paths(input_folder, 1, 'tif'):
        if init_test in path:
            init_image_list.append(path)
        if final_test in path:
            final_image_list.append(path)
    composit_image_list.append(fullpath(output_folder, test['composit'], 'tif'))

argv = sys.argv

if argv is None:
    sys.exit(1)

if not os.path.exists(output_folder):
    os.makedirs(output_folder)

'''
# Parse command line arguments.
i = 1
while i < len(argv):
    arg = argv[i]

    if init_image is None:
        init_image = argv[i]

    elif final_image is None:
        final_image = argv[i]

    elif output_folder is None:
        output_folder = argv[i]

    elif arg == '-io':
        i = i + 1
        images_order = [j for j in argv[i].split(',')]

    elif arg == '-bo':
        i = i + 1
        bands_order = [int(j) for j in argv[i].split(',')]

    else:
        Usage()

    i = i + 1

if init_image is None:
    Usage()

if final_image is None:
    Usage()

if output_folder is None:
    Usage()
'''

error_list = []

for init_image, final_image, composit_image in zip(init_image_list, final_image_list, composit_image_list):

        imI_wkt, imI_epsg, imI_srs, imI_gt = ImageProperties(init_image)
        imF_wkt, imF_epsg, imF_srs, imF_gt = ImageProperties(final_image)
        imI_geom = ogr.CreateGeometryFromWkt(imI_wkt, imI_srs)
        imF_geom = ogr.CreateGeometryFromWkt(imF_wkt, imF_srs)
        intersection_geom = imI_geom.Intersection(imF_geom)
        overlap = round(100 * intersection_geom.Area() / imI_geom.Area(), 2)

        if overlap>0:
            if os.path.exists(composit_image):
                print('File already exists: {}'.format(composit_image))
                continue
            t_composit_image = '%s_t.tif' % composit_image[:-4]
            t_corregistred_image = '%s_t_corrected.tif' % composit_image[:-4]
            init_image_sname = os.path.basename(init_image)
            final_image_sname = os.path.basename(final_image)

            try:

                if not os.path.exists(t_corregistred_image):
                    pci_auto_coregistration(final_image, init_image, t_corregistred_image,
                        in_match_chan = [bands_order[images_order.index('f')]],
                        ref_match_chan = [bands_order[images_order.index('i')]],
                        bxpxsz=str(imI_gt[1]), polyorder=[1], reject = [5, 1, 1], tresh=[0.88])

                if os.path.exists(t_corregistred_image):
                    imC_wkt, imC_epsg, imC_srs, imC_gt = ImageProperties(t_corregistred_image)
                    imC_geom = ogr.CreateGeometryFromWkt(imC_wkt, imC_srs)
                    intersection_geom = imI_geom.Intersection(imC_geom)
                    ulx, lrx, lry, uly = intersection_geom.GetEnvelope()
                    bbox=[str(ulx),str(uly),str(lrx),str(lry)]
                    TemporyInitImage=t_composit_image.replace('.tif','_init.VRT')
                    TemporyFinImage=t_composit_image.replace('.tif','_final.VRT')
                    cmd='gdal_translate -of VRT -projwin {0} {1} {2}'.format (' '.join(bbox), init_image, TemporyInitImage)
                    runcmd(cmd)
                    cmd='gdal_translate -of VRT -projwin {0} {1} {2}'.format(' '.join(bbox), t_corregistred_image, TemporyFinImage)
                    runcmd(cmd)
                    imageI_sub=gdal.Open( TemporyInitImage, gdalconst.GA_ReadOnly )
                    imageF_sub=gdal.Open( TemporyFinImage, gdalconst.GA_ReadOnly )
                    used_bands = []

                    for b in bands_order:
                        if b not in used_bands:
                            used_bands+=[b]

                    mask = BuildActiveAreaMask([imageI_sub, imageF_sub], bands_order)

                    if not os.path.exists(t_composit_image):
                        dst_ds = gdal.GetDriverByName("GTiff").Create( t_composit_image , mask.shape[1], mask.shape[
                            0],  len(bands_order), gdalconst.GDT_UInt16)
                        dst_ds.SetGeoTransform( imageI_sub.GetGeoTransform() )
                        dst_ds.SetProjection( imageI_sub.GetProjection() )

                        for image, band, band_index in zip(images_order, bands_order, range(1, len(bands_order)+1)):

                            if image=='f':
                                band_data = imageF_sub.GetRasterBand(band).ReadAsArray().astype(np.uint16)

                            elif image=='i':
                                band_data = imageI_sub.GetRasterBand(band).ReadAsArray().astype(np.uint16)

                            band_data[mask==0]=0
                            dst_ds.GetRasterBand(band_index).WriteArray(band_data)

                        dst_ds=None
                        GeoTiffSetNodata(t_composit_image, 0)

                    imageF_sub = None
                    imageI_sub = None

                    print(t_composit_image, composit_image)
                    copydeflate(t_composit_image, composit_image)

                    SaveRasterBands(composit_image, [5,1,6], composit_image[:-4]+'_3CH.tif', options={'compress': 'DEFLATE'}, overwrite=False)

                    for i_f in [TemporyInitImage, TemporyFinImage, t_corregistred_image, t_composit_image]:
                        if os.path.exists(i_f):
                            os.remove(i_f)

                else:
                    print('Corregistration error')

            except:
                print('Error making file {}'.format(composit_image))
                error_list.append(composit_image)

        else:
            print('Overlaping error - %s '%str(overlap))

scroll(error_list, header='Errors found')


import math
import random
import string
import asyncio

first_value = -math.pi
second_value = -math.pi

async def calculate_sin(var_ref):
    global first_value
    await var_ref.set_value(math.sin(first_value))
    first_value += 0.1
    if first_value > math.pi:
        first_value = -math.pi

async def calculate_cos(var_ref):
    global second_value
    await var_ref.set_value(math.cos(second_value))
    second_value += 0.1
    if second_value > math.pi:
        second_value = -math.pi

async def increaser(var_ref, step):
    await var_ref.set_value(await var_ref.get_value() + step)

async def text_generator(var_ref, length=16):
    await var_ref.set_value(''.join(random.choice(string.ascii_lowercase) for i in range(length)))

async def update_vars(vars_ref):
    while True:
        await calculate_sin(vars_ref[0])
        await increaser(vars_ref[1], 1)
        await increaser(vars_ref[2], -1)
        await text_generator(vars_ref[3])
        await calculate_cos(vars_ref[4])
        await calculate_sin(vars_ref[5])
        await increaser(vars_ref[6], 5)
        await increaser(vars_ref[7], -0.1)
        await calculate_cos(vars_ref[8])
        await text_generator(vars_ref[9], 20)
        await increaser(vars_ref[10], 2)
        await asyncio.sleep(1)


from getpass import getuser
from os import environ
from pathlib import Path

from .config import configuration

def restart_count():
    """
    How many times has this job restarted?
    Writes to a file in the logging directory in order to record
    the number of restarts. Grid Engine will tell a task that it
    has restarted but not how many times. This makes a file
    with Job ID and Task Id and the ending ``.restart``, with
    one character in it per restart.

    Returns:
        int: The number of restarts.
    """
    try:
        clipped_restart_cnt = int(environ.get("RESTARTED", 0))
    except ValueError:
        clipped_restart_cnt = 0
    if clipped_restart_cnt == 0:
        restart_cnt = 0
    else:
        template = configuration()["restart-file-location"]
        temporary_path = Path(template.format(user=getuser()))
        temporary_path.mkdir(parents=True, exist_ok=True)
        job_id = [environ.get("JOB_ID", "unknown-job")]
        task_id = environ.get("SGE_TASK_ID", "undefined")
        if task_id != "undefined":
            job_id.append(task_id)
        marker_path = temporary_path / "_".join(job_id)
        if marker_path.exists():
            with marker_path.open("r") as check:
                restart_cnt = len(check.read()) + 1
        else:
            restart_cnt = 1
        with marker_path.open("a") as mark:
            mark.write(".")

    return restart_cnt


import os
from dotenv import load_dotenv
import discord
from discord.ext import commands, tasks
from datetime import datetime
import variables
from data.items import categories
from helpers.auth import is_authorized
from helpers.notifications import send_rogue_error_webhook
import logging

# Start logging to discord.log file
logger = logging.getLogger('discord')
logger.setLevel(logging.DEBUG)
handler = logging.FileHandler(filename='discord.log', encoding='utf-8', mode='w')
handler.setFormatter(logging.Formatter('%(asctime)s:%(levelname)s:%(name)s: %(message)s'))
logger.addHandler(handler)

# Load environment variables
load_dotenv(override=True)
TOKEN = os.getenv('DISCORD_TOKEN')

# Insert names of cogs that you don't want to initialize at startup (default = all cogs loaded)
DO_NOT_LOAD_COGS_AT_STARTUP = []

# Initialize Discord bot and add global command check from auth
client = commands.Bot(command_prefix=variables.command_prefix)
client.add_check(is_authorized)

# Set max length per check (If check time exceeds this time, it will send an error and stop tracking)
max_length_per_check = 300

# When bot is loaded and ready
@client.event
async def on_ready():
    # Set bot presence
    await client.change_presence(activity=discord.Activity(type=discord.ActivityType.watching,
                                                           name=f'Fitness Stock | {variables.command_prefix}help'))
    print(f'{client.user} has connected to Discord and is ready!')

    # Assign notify role ids and webhooks for all categories based on .env file
    # Set .env file variables to:
    # <CATEGORY>-role-id=<ROLE ID TO TAG FOR THIS ITEM> (If unassigned, it will tag @everyone when notifies are on)
    # <CATEGORY>-webhook=<WEBHOOK URL TO POST STOCK UPDATE TO> (If unassigned, it will post to ROGUE_FITNESS_WEBHOOK_URL env variable
    if len(variables.rogue_category_data) <= 0:
        for category in categories:
            try:
                role_id = int(os.getenv(f'{category}-role-id'))
            except:
                role_id = None
            variables.rogue_category_data[category] = {
                'notify_role': discord.utils.find(lambda m: m.id == role_id, client.guilds[0].roles),
                'webhook_url': os.getenv(f'{category}-webhook')
            }
        print(f'Category Tracking: {variables.rogue_category_data}')

    # Assign bot manager id based on .env file
    # Set .env file variable to:
    # bot-manager-id=<ROLE ID FOR BOT MANAGER TO TAG ON ERRORS> (If unassigned, will not tag anyone)
    if variables.bot_manager is None:
        try:
            bot_manager_id = int(os.getenv('bot-manager-id'))
        except:
            bot_manager_id = None
        variables.bot_manager = discord.utils.find(lambda m: m.id == bot_manager_id, client.guilds[0].roles)
        if variables.bot_manager is not None:
            print(f'Set Bot Manager to: {variables.bot_manager.name} (Role ID: {variables.bot_manager.id})')
        else:
            print(f'No Bot Manager Set (use {variables.command_prefix}authrole to set a role)')

    # If bot lost connection, send status update to inform bot tracking has restarted
    if variables.is_tracking_rogue:
        send_rogue_error_webhook(f'ERROR #1: Cloud Server connection error. Bot reconnected and Rogue tracking has '
                                 f'automatically been restarted. Verify checks with {variables.command_prefix}status '
                                 f'or by viewing console logs.', stop_tracking=False)

# Compare current check time length every 60 seconds with max_length_per_check
# and if it exceeds, send error and stop tracking.
@tasks.loop(seconds=60)
async def verify_rogue_tracking_integrity():
    if not variables.is_tracking_rogue:
        return
    else:
        if variables.last_successful_check_datetime is not None:
            time_difference = datetime.now() - variables.last_successful_check_datetime
            print(f'\tTime Difference since last successful check: {time_difference.total_seconds()}')
            if time_difference.total_seconds() > max_length_per_check:
                send_rogue_error_webhook(f'ERROR #0: Cloud Server connection error. Rogue check script has frozen '
                                         f'with {time_difference.total_seconds()} seconds since last successful check '
                                         f'at {variables.last_successful_check}. Please restart Rogue tracking with '
                                         f'{variables.command_prefix}rogue.')
                variables.last_successful_check_datetime = None

# Command error handler
@client.event
async def on_command_error(ctx, error):
    if isinstance(error, commands.MissingPermissions):
        await ctx.message.delete()
        await ctx.send("You don't have the permissions to run this command!")
    elif isinstance(error, commands.MissingRole):
        await ctx.message.delete()
        await ctx.send("You don't have the proper role(s) to run this command!")
    elif isinstance(error, commands.MissingRequiredArgument):
        await ctx.message.delete()
        await ctx.send('Please pass in all required arguments.')
    else:
        print(f'{type(error)} - {error}')

# Logout and close all bot connections to Discord
@client.command(aliases=['logout', 'close', 'disconnect', 'exit'],
                brief='Take bot offline for maintenance and send message. (ADMIN)')
async def maintenance(ctx):
    await ctx.message.delete()
    await ctx.send(f'Bot going offline for maintenance and updates.')
    await client.close()

# Authorize role by id or name to use bot commands
@client.command(brief='Gets or sets authorized bot manager role. (ADMIN)')
async def authrole(ctx, role: discord.Role = None):
    await ctx.message.delete()
    if role is None:
        if variables.bot_manager is None:
            await ctx.send(f'No authorized bot manager role is currently assigned.')
        else:
            await ctx.send(f'Current authorized bot manager role set to: {variables.bot_manager.mention}')
    else:
        variables.bot_manager = role
        await ctx.send(f'Set the authorized bot manager role to: {role.mention}')

# Specific error handler for authrole command
@authrole.error
async def role_error(ctx, error):
    if isinstance(error, discord.ext.commands.BadArgument):
        await ctx.message.delete()
        await ctx.send('Could not find that role.')

# Load cog specified in command
@client.command(brief='Loads specified extension (ADMIN)')
async def load(ctx, extension):
    await ctx.message.delete()
    try:
        client.load_extension(f'cogs.{extension}')
    except commands.ExtensionError as e:
        await ctx.send(f'**`ERROR:`** {e.name} - {e}')
    else:
        await ctx.send(f'{extension.capitalize()} cog has been loaded.')
        print(f'{extension.capitalize()} cog has been loaded.')

# Unload cog specified in command
@client.command(brief='Unloads specified extension (ADMIN)')
async def unload(ctx, extension):
    await ctx.message.delete()
    try:
        client.unload_extension(f'cogs.{extension}')
    except commands.ExtensionError as e:
        await ctx.send(f'**`ERROR:`** {e.name} - {e}')
    else:
        await ctx.send(f'{extension.capitalize()} cog has been unloaded.')
        print(f'{extension.capitalize()} cog has been unloaded.')

# Reload cog specified in command
@client.command(brief='Reloads specified extension (ADMIN)')
async def reload(ctx, extension=None):
    await ctx.message.delete()
    if extension is None:
        reload_message = await ctx.send(f'Reloading all cogs.')
        for filename in os.listdir('./cogs'):
            if f'{filename.split(".", 1)[0]}' in DO_NOT_LOAD_COGS_AT_STARTUP:
                continue
            if filename.endswith('.py'):
                extension_name = filename.split(".", 1)[0].capitalize()
                cog_extension = f'cogs.{filename[:-3]}'
                try:
                    client.unload_extension(cog_extension)
                except:
                    client.load_extension(cog_extension)
                else:
                    client.load_extension(cog_extension)
                await ctx.send(f'{extension_name} cog has been reloaded.')
        await reload_message.delete()
    else:
        try:
            client.unload_extension(f'cogs.{extension}')
            client.load_extension(f'cogs.{extension}')
        except commands.ExtensionError as e:
            await ctx.send(f'**`ERROR:`** {e.name} - {e}')
        else:
            await ctx.send(f'{extension.capitalize()} cog has been reloaded.')
            print(f'{extension.capitalize()} cog has been reloaded.')

# Load all cogs initially except those in the DO_NOT_LOAD_COGS_AT_STARTUP list
for filename in os.listdir('./cogs'):
    if f'{filename.split(".", 1)[0]}' in DO_NOT_LOAD_COGS_AT_STARTUP:
        continue
    if filename.endswith('.py'):
        client.load_extension(f'cogs.{filename[:-3]}')

# Start verify_rogue_tracking_integrity task
try:
    verify_rogue_tracking_integrity.start()
    variables.rogue_integrity_check = True
except Exception as e:
    variables.rogue_integrity_check = False
    send_rogue_error_webhook(f'{type(e)} - {e}: Unable to start Rogue tracking integrity. Please check console logs '
                             f'and restart Rogue tracking with {variables.command_prefix}rogue.')

# Start bot
client.run(TOKEN)


from PyQt6.QtWidgets import *
from PyQt6.QtGui import *
from mycanvas import *
from mymodel import *

class MyWindow(QMainWindow):

    def __init__(self):
        super(MyWindow, self).__init__()
        self.setGeometry(100,100,600,400)
        self.setWindowTitle("MyGLDrawer")
        self.canvas = MyCanvas()
        self.setCentralWidget(self.canvas)
        # create a model object and pass it to canvas object
        self.model = MyModel()
        self.canvas.setModel(self.model)
        # create a Toolbar
        tb = self.addToolBar("File")
        fit = QAction(QIcon("curve_colector/icons/fit.jpg"),"fit",self)
        tb.addAction(fit)
        tb.actionTriggered[QAction].connect(self.tbpressed)
#        tb2 = self.addToolBar("Draw")
#        line = QAction(QIcon("icons/fit.jpg"),"line",self)
#        tb2.addAction(line)

    def tbpressed(self,a):
        if a.text() == "fit":
            self.canvas.fitWorldToViewport()


from typing import List
from pandas import DataFrame

from domain.contracts.abstract_results_storage_service import AbstractResultsStorageService
from domain.models.inference_object import InferenceObject
from domain.models.evaluation_job_results import EvaluationJobResults
from domain.excpetions.validation_exceptions import InvalidJobId

class ResultsStorageService(AbstractResultsStorageService):

    def __init__(self, results_retrieval_service, results_repository):
        self.results_retrival_service = results_retrieval_service
        self.results_repository = results_repository

    def save_predictions(self, uid: str, predictions_to_save: List[InferenceObject]) -> None:
        evaluation_job_results: EvaluationJobResults = self.results_retrival_service.get_results(uid)
        for prediction in predictions_to_save:
            evaluation_job_results.predictions = evaluation_job_results.predictions.append(prediction.dict(),
                                                                                ignore_index=True)

        self.results_repository.save_predictions(uid, evaluation_job_results)

    def save_ground_truths(self, uid: str, predictions_to_save: List[InferenceObject]) -> None:
        evaluation_job_results: EvaluationJobResults = self.results_retrival_service.get_results(uid)
        for prediction in predictions_to_save:
            evaluation_job_results.ground_truths = evaluation_job_results.ground_truths.append(prediction.dict(),
                                                                                ignore_index=True)
        self.results_repository.save_ground_truths(uid, evaluation_job_results)

    def save_linkages(self, uid: str, linkages_to_save: DataFrame) -> None:
        self.results_repository.save_linkages(uid, linkages_to_save)

    def get_results_based_on_image_paths(self, uid: str, image_paths: List[str]) -> EvaluationJobResults:
        if not uid in self.results_repository.store.keys():
            self.results_repository.store[uid] = EvaluationJobResults(
                ground_truths=DataFrame(),
                predictions=DataFrame(),
                base_linkages=DataFrame(),
                classification_linkages=DataFrame(),
                linkages_by_iou=DataFrame(),
                inference_time=0
            )
        results: EvaluationJobResults = self.results_repository.get_results(uid)

        results.predictions = results.predictions.loc[results.predictions.image_path.isin(image_paths)]
        results.ground_truths = results.ground_truths.loc[results.ground_truths.image_path.isin(image_paths)]
        return results

    def delete_results(self, uid: str) -> None:

        try:
            self.results_repository.delete_results(uid)

        except InvalidJobId:
            raise InvalidJobId


import numpy as np
import PltFuncs as p
import ipdb
import pylab as plt

def KappaVsM1(kappaList, NList = [80000], nTrials=10, T=1000, K=1000):
    # saves x and y in pnas/data
    for N in NList:
        p.KappaVsM1(kappaList, mExtOne=0, IF_COMPUTE=1, T=T, nTrials=nTrials, nPhis=1, N=N, K=K)

if __name__ == "__main__":
    # T=4000
    # kappaList = np.arange(0, 10, 0.1)
    kappaList = np.array([8.2, 8.6, 8.8, 9.0])
    # kappaList = np.array([8.8])
    # kappaList = np.array([4, 4.1, 4.2, 4.3, 4.4, 4.5, 4.6, 4.7, 4.8, 4.9, 5])
    # kappaList = np.array([7.5, 7.6, 7.7, 7.8, 7.9, 8.0, 8.1, 8.2])
    nTrials = 10
    NList = [20000] #, 80000] #, 20000, 40000]
    K = 2000

    for T in [2000, 4000]:
        KappaVsM1(kappaList, NList, nTrials, T=T, K=K)
        p.KappaVsM1Plot(NList, kappaList, T=T, K=K)
        #p.KappaVsM1PlotPoints(NList, kappaList, T=T)

    # plt.ion()
    # plt.show()
    # ipdb.set_trace()


import pytest
import json

pytestmark = pytest.mark.django_db

class TestCategoryEndpoints:
    endpoint = "/api/category/"

    def test_category_get(self, category_factory, api_client):
        # Arrange
        category_factory.create_batch(4)
        # Act
        response = api_client().get(self.endpoint)
        # Assert
        assert response.status_code == 200
        assert len(json.loads(response.content)) == 4

class TestBrandEndpoints:
    endpoint = "/api/brand/"

    def test_brand_get(self, brand_factory, api_client):
        # Arrange
        brand_factory.create_batch(4)
        # Act
        response = api_client().get(self.endpoint)
        # Assert
        assert response.status_code == 200
        assert len(json.loads(response.content)) == 4

class TestProductEndpoints:
    endpoint = "/api/product/"

    def test_product_get(self, product_factory, api_client):
        # Arrange
        product_factory.create_batch(4)
        # Act
        response = api_client().get(self.endpoint)
        # Assert
        assert response.status_code == 200
        assert len(json.loads(response.content)) == 4


import random, pygame, sys

pygame.init()
y = 10  # int(input('y = '))
x = 10  # int(input('x = '))
n = 10  # int(input('n = '))
Arial = pygame.font.SysFont('Arial', 20)
win = pygame.display.set_mode((x * 20, y * 20))
img = pygame.image.load('img.png')
bombs = []
bmap = []
fieldss = []
bombss = []
fields = []
flag = pygame.image.load('flag.png')
clock = pygame.time.Clock()
showntiles = []
prevclick = 0

def map_gen(x, y, n):#generates map as a 2d list
    global bombs
    global bmap
    global map
    map = []
    bombs = []
    bmap = []

    for i in range(x):
        row = []
        for j in range(y):
            bombs.append([i, j])
            row.append(0)
        bmap.append(row)

    for i in range(n):
        bomb = random.choice(bombs)

        bombs.remove(bomb)
        bmap[(bomb[0] - 1)][(bomb[1] - 1)] = "b"

    print("done")

print(y)
print(x)
map_gen(x, y, n)

a = -1
for i in bmap: #creates a list of bombs' coordintes
    b = 0
    a += 1
    for j in i:
        if j == 'b':
            bombss.append([a, b])
        b += 1

print(bombss)

for i in bombss:  # creates number indicating the ammount of mines/bombs in neighbourring tiles
    if not i[0] == 0 and not i[1] == 0 and not i[0] == (y - 1) and not i[1] == (x - 1):
        if not bmap[i[0] - 1][i[1] - 1] == 'b':
            bmap[i[0] - 1][i[1] - 1] += 1
        if not bmap[i[0] - 1][i[1]] == 'b':
            bmap[i[0] - 1][i[1]] += 1
        if not bmap[i[0] - 1][i[1] + 1] == 'b':
            bmap[i[0] - 1][i[1] + 1] += 1
        if not bmap[i[0]][i[1] - 1] == 'b':
            bmap[i[0]][i[1] - 1] += 1
        if not bmap[i[0]][i[1] + 1] == 'b':
            bmap[i[0]][i[1] + 1] += 1
        if not bmap[i[0] + 1][i[1] - 1] == 'b':
            bmap[i[0] + 1][i[1] - 1] += 1
        if not bmap[i[0] + 1][i[1]] == 'b':
            bmap[i[0] + 1][i[1]] += 1
        if not bmap[i[0] + 1][i[1] + 1] == 'b':
            bmap[i[0] + 1][i[1] + 1] += 1

    elif i[0] == 0 and not i[1] == 0 and not i[1] == (x - 1):
        if not bmap[i[0]][i[1] - 1] == 'b':
            bmap[i[0]][i[1] - 1] += 1
        if not bmap[i[0]][i[1] + 1] == 'b':
            bmap[i[0]][i[1] + 1] += 1
        if not bmap[i[0] + 1][i[1] - 1] == 'b':
            bmap[i[0] + 1][i[1] - 1] += 1
        if not bmap[i[0] + 1][i[1]] == 'b':
            bmap[i[0] + 1][i[1]] += 1
        if not bmap[i[0] + 1][i[1] + 1] == 'b':
            bmap[i[0] + 1][i[1] + 1] += 1

    elif i[0] == 0 and i[1] == 0:
        if not bmap[i[0]][i[1] + 1] == 'b':
            bmap[i[0]][i[1] + 1] += 1
        if not bmap[i[0] + 1][i[1]] == 'b':
            bmap[i[0] + 1][i[1]] += 1
        if not bmap[i[0] + 1][i[1] + 1] == 'b':
            bmap[i[0] + 1][i[1] + 1] += 1

    elif not i[0] == 0 and i[1] == 0 and not i[0] == y - 1:
        if not bmap[i[0] - 1][i[1]] == 'b':
            bmap[i[0] - 1][i[1]] += 1
        if not bmap[i[0] - 1][i[1] + 1] == 'b':
            bmap[i[0] - 1][i[1] + 1] += 1
        if not bmap[i[0]][i[1] + 1] == 'b':
            bmap[i[0]][i[1] + 1] += 1
        if not bmap[i[0] + 1][i[1]] == 'b':
            bmap[i[0] + 1][i[1]] += 1
        if not bmap[i[0] + 1][i[1] + 1] == 'b':
            bmap[i[0] + 1][i[1] + 1] += 1
    elif i[0] == y and i[1] == 0:
        if not bmap[i[0] - 1][i[1]] == 'b':
            bmap[i[0 - 1]][i[1]] += 1
        if not bmap[i[0] + 1][i[1] + 1] == 'b':
            bmap[i[0] + 1][i[1] + 1] += 1
        if not bmap[i[0]][i[1] + 1] == 'b':
            bmap[i[0]][i[1] + 1] += 1
    elif i[0] == (y - 1) and i[1] == 0:
        if not bmap[i[0] - 1][i[1]] == 'b':
            bmap[i[0] - 1][i[1]] += 1
        if not bmap[i[0] - 1][i[1] + 1] == 'b':
            bmap[i[0] - 1][i[1] + 1] += 1
        if not bmap[i[0]][i[1] + 1] == 'b':
            bmap[i[0]][i[1] + 1] += 1
    elif i[0] == (y - 1) and not i[1] == 0 and not i[1] == (x - 1):
        if not bmap[i[0] - 1][i[1] - 1] == 'b':
            bmap[i[0] - 1][i[1] - 1] += 1
        if not bmap[i[0] - 1][i[1]] == 'b':
            bmap[i[0] - 1][i[1]] += 1
        if not bmap[i[0] - 1][i[1] + 1] == 'b':
            bmap[i[0] - 1][i[1] + 1] += 1
        if not bmap[i[0]][i[1] - 1] == 'b':
            bmap[i[0]][i[1] - 1] += 1
        if not bmap[i[0]][i[1] + 1] == 'b':
            bmap[i[0]][i[1] + 1] += 1
    elif i[0] == (y - 1) and i[1] == (x - 1):
        if not bmap[i[0] - 1][i[1] - 1] == 'b':
            bmap[i[0] - 1][i[1] - 1] += 1
        if not bmap[i[0] - 1][i[1]] == 'b':
            bmap[i[0] - 1][i[1]] += 1
        if not bmap[i[0]][i[1] - 1] == 'b':
            bmap[i[0]][i[1] - 1] += 1
    elif not i[0] == (y - 1) and not i[0] == 0 and i[1] == (x - 1):
        if not bmap[i[0] - 1][i[1] - 1] == 'b':
            bmap[i[0] - 1][i[1] - 1] += 1
        if not bmap[i[0] - 1][i[1]] == 'b':
            bmap[i[0] - 1][i[1]] += 1
        if not bmap[i[0]][i[1] - 1] == 'b':
            bmap[i[0]][i[1] - 1] += 1
        if not bmap[i[0] + 1][i[1] - 1] == 'b':
            bmap[i[0] + 1][i[1] - 1] += 1
        if not bmap[i[0] - 1][i[1]] == 'b':
            bmap[i[0] - 1][i[1]] += 1
    elif i[0] == y and i[1] == (x - 1):
        if not bmap[i[0]][i[1] - 1] == 'b':
            bmap[i[0]][i[1] - 1] += 1
        if not bmap[i[0] + 1][i[1] - 1] == 'b':
            bmap[i[0] + 1][i[1] - 1] += 1
        if not bmap[i[0] + 1][i[1]] == 'b':
            bmap[i[0] + 1][i[1]] += 1

class tile(object):
    def __init__(self, x, y, isb, ishid, isflagged):
        global bmap
        self.x = x
        self.y = y
        self.isb = isb
        self.ishid = ishid
        self.isflagged = isflagged
        self.val = bmap[y][x]

    def draw(self, win):

        if self.ishid == False and self.isflagged == False:
            if self.isb == True:
                win.blit(img, ([self.x * 20, self.y * 20]))
            else:
                num = Arial.render(str(bmap[self.y][self.x]), 1, (0, 0, 255))
                win.blit(num, (self.x * 20 + 3, self.y * 20))
        elif self.isflagged == True:
            win.blit(flag, ([self.x * 20, self.y * 20]))

        else:
            pygame.draw.rect(win, (127 * (self.x - 2 * (self.x // 2) + self.y - 2 * (self.y // 2)), #creates a visible crosword-word-puzzle-like shape so we can tell where where given tiles are
                                   127 * (self.x - 2 * (self.x // 2) + self.y - 2 * (self.y // 2)),
                                   127 * (self.x - 2 * (self.x // 2) + self.y - 2 * (self.y // 2))),
                             (20 * self.x, 20 * self.y, 20, 20))

    def odkryj(self): #shows content of a given tile (number or an image of a bomb(unless its flagged, then it just removes the flag)
        if self.isflagged == False:
            if self.ishid == True: self.ishid = False

        else:
            self.isflagged = False

    def flagg(self):
        if self.ishid == True: self.isflagged = True

    def value(self):
        return self.val

for i in range(y):

    for j in range(x):
        if bmap[i][j] == 'b':
            fields.append(tile(j, i, True, True, False))
        else:
            fields.append(tile(j, i, False, True, False))

# print(fieldss)
def redrawgamewindow(): #displays the whole thing
    for field in fields:
        field.draw(win)

    pygame.display.update()

run = True
while run:  # mainloop
    redrawgamewindow()
    mouse = pygame.mouse.get_pos()
    click = pygame.mouse.get_pressed()

    if prevclick == 0 and click[0] == 1:
        fields[int((mouse[1] // 20) * x + (mouse[0] // 20))].odkryj()
    if click[2] == 1:
        fields[int((mouse[1] // 20) * x + (mouse[0] // 20))].flagg()

    prevclick = click[0]
    for event in pygame.event.get(): #makes the X button kill the program
        if event.type == pygame.QUIT:
            run = False
pygame.quit()


# Create an elbow curve to find the best value for K.
inertia = []
k = list(range(1, 11))

for i in k:
    km = KMeans(n_clusters=i, random_state=0)
    km.fit(df_crypto_scaled_pca)
    inertia.append(km.inertia_)

elbow_data = {"k": k, "inertia": inertia}
df_elbow = pd.DataFrame(elbow_data)
df_elbow.hvplot.line(
    x="k",
    y="inertia",
    xticks=k,
    title="Elbow Visualization",
    xlabel="Number of Clusters",
    ylabel="Inertia",
)


from pdf2image import convert_from_path
from argparse import ArgumentParser

parser = ArgumentParser()
parser.add_argument("file", help="分割したいPDFファイル")
args = parser.parse_args()

file = args.file

pages = convert_from_path(file)
for i, page in enumerate(pages):
    filename = f"{i+1:02}.jpg"
    page.save(filename)


from prometheus_client import Counter
from prometheus_client import Gauge
from prometheus_client import Info
from prometheus_client import CollectorRegistry

# separate registry is used, as the default registry may contain other metrics
registry = CollectorRegistry()
g_reg = CollectorRegistry()
# Define Counters
# Doc: https://github.com/prometheus/client_python
# Define daemon log counters
# DHCPS events
daemon_c_dhcps = Counter('daemon_c_dhcps',
                         'Number of DHCP request in daemon log',
                         registry=registry)
daemon_c_dhcps_ack = Counter('daemon_c_dhcps_ack',
                             'Number of DHCP ack in daemon log',
                             registry=registry)
daemon_c_dhcps_req = Counter('daemon_c_dhcps_req',
                             'Number of DHCP  req in daemon log',
                             registry=registry)
daemon_c_dhcps_offer = Counter('daemon_c_dhcps_offer',
                               'Number of DHCP offers in daemon log',
                               registry=registry)
daemon_dhcp_info = Info('daemon_dhcp_info',
                        'Daemon log Info about DHCP activities',
                        registry=registry)
# Auth events
daemon_c_auth = Counter('daemon_c_auth',
                        'Number of Auth request in daemon log',
                        registry=registry)

# FIREWALL Events
daemon_c_fw = Counter('daemon_c_fw',
                      'Number of Firewall alarts in daemon log',
                      registry=registry)
daemon_c_fw_tcp = Counter('daemon_c_fw_tcp',
                          'Number of Firewall TCP alarts in daemon log',
                          registry=registry)
daemon_c_fw_udp = Counter('daemon_c_fw_udp',
                          'Number of Firewall UDP alarts in daemon log',
                          registry=registry)
daemon_c_fw_icmp = Counter('daemon_c_fw_icmp',
                           'Number of Firewall ICMP alarts in daemon log',
                           registry=registry)
daemon_g_fw = Gauge('daemon_g_fw',
                    'Number of Firewall alarts in daemon log',
                    registry=g_reg)
daemon_info = Info('info_daemon',
                   'Daemon log Info',
                   registry=registry)
daemon_fw_info = Info('daemon_fw_info',
                      'Daemon log Info about FW activities',
                      registry=registry)

# Define user log counters
user_c_auth = Counter('user_c_auth',
                      'Number of Auth request in user log',
                      registry=registry)
user_g_auth = Gauge('user_g_auth',
                    'Number of Auth request in user log',
                    registry=g_reg)
user_c_sys = Counter('user_sys',
                     'Number of System alarts in user log',
                     registry=registry)
user_g_sys = Gauge('user_g_sys',
                   'Number of System alarts in user log',
                   registry=g_reg)
user_info = Info('user_info',
                 'User log Info',
                 registry=registry)
# Define corn log counters
# No counters at this time
cron_c_count = Counter('cron_c_count_cron',
                       'Number of alarts in cron log',
                       registry=registry)
cron_g_count = Gauge('cron_g_count',
                     'Number of alarts in cron log',
                     registry=g_reg)
cron_info = Info('cron_info_cron',
                 'Cron log Info',
                 registry=registry)
# Other Counter
g_unknwon_count = Gauge('unknwon_count',
                        'Number of Unknown events in log',
                        registry=g_reg)
unknwon_count = Counter('unknwon_count',
                        'Number of Unknown events in log',
                        registry=registry)


"""Misc."""
import collections
import logging
import os

import toml

def merge_toml_env(old_env, mods):
    """Merge."""
    return deep_update(old_env, mods)

def merge_toml_env_from_files(toml_files):
    """Merge."""
    merged_env = {}
    for toml_file in toml_files:
        if os.path.exists(toml_file):
            with open(toml_file, mode="r", encoding="utf-8") as file_handler:
                modification = toml.load(file_handler)
            merged_env = merge_toml_env(merged_env, modification)
        else:
            print("WARNING: File not found " + toml_file)
    return merged_env

def deep_update(source, overrides):
    """Update a nested dictionary or similar mapping.

    Modify ``source`` in place.

    Args:
        source(dict): Source data
        overrides(dict): Delta data to override

    Returns:
        source(dict): Updated dict
    """
    for key, value in overrides.items():
        if isinstance(value, collections.abc.Mapping) and value:
            returned = deep_update(source.get(key, {}), value)
            source[key] = returned
        else:
            override = overrides[key]
            source[key] = override
    return source

def remove_existing_file(f_in, f_out):
    """Remove existing file.

    Args:
        f_in (_type_): _description_
        f_out (_type_): _description_

    Raises:
        FileNotFoundError: _description_
        IsADirectoryError: _description_

    """
    if f_in is None:
        raise FileNotFoundError("Input file not set")
    # If files are not the same file
    if os.path.abspath(f_in) != os.path.abspath(f_out):
        if os.path.isdir(f_out):
            raise IsADirectoryError(
                f_out + " is a directory! Please remove it if desired"
            )
        if os.path.islink(f_out):
            os.unlink(f_out)
        if os.path.isfile(f_out):
            os.remove(f_out)
    # files have the same path. Remove if it is a symlink
    else:
        if os.path.islink(f_out):
            os.unlink(f_out)

def parse_filepattern(file_pattern, basetime, validtime):
    """Parse the file pattern.

    Args:
        file_pattern (str): File pattern.
        basetime (datetime.datetime): Base time.
        validtime (datetime.datetime): Valid time.

    Returns:
        str: File name

    """
    if basetime is None or validtime is None:
        return file_pattern

    logging.debug(
        "file_pattern=%s basetime=%s validtime=%s", file_pattern, basetime, validtime
    )
    file_name = str(file_pattern)
    year = basetime.strftime("%Y")
    year2 = basetime.strftime("%y")
    month = basetime.strftime("%m")
    day = basetime.strftime("%d")
    hour = basetime.strftime("%H")
    mins = basetime.strftime("%M")
    d_t = validtime - basetime
    ll_d = f"{int(d_t.seconds / 3600):d}"
    ll_2 = f"{int(d_t.seconds / 3600):02d}"
    ll_3 = f"{int(d_t.seconds / 3600):03d}"
    ll_4 = f"{int(d_t.seconds / 3600):04d}"
    file_name = file_name.replace("@YYYY@", year)
    file_name = file_name.replace("@YY@", year2)
    file_name = file_name.replace("@MM@", month)
    file_name = file_name.replace("@DD@", day)
    file_name = file_name.replace("@HH@", hour)
    file_name = file_name.replace("@mm@", mins)
    file_name = file_name.replace("@L@", ll_d)
    file_name = file_name.replace("@LL@", ll_2)
    file_name = file_name.replace("@LLL@", ll_3)
    file_name = file_name.replace("@LLLL@", ll_4)
    return file_name


import collections
import json
import time
import traceback
import httpx
import sys
import asyncio
import inspect
try:
    import uvloop
    uvloop_installed = True
except ImportError:
    uvloop = None
    uvloop_installed = False

from aio_pool import AioPool

USERAGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/109.0'

async def sure(fn, max_sleep=32):
    t = 1
    while True:
        try:
            return await fn()
        except Exception:
            print(f'sure failed', inspect.getsource(fn).strip())
            traceback.print_exc()
            await asyncio.sleep(t)
            t = min(max_sleep, t * 2)

def find_dicts_deep(data, fn_match):
    l = []
    if isinstance(data, dict):
        if fn_match(data):
            l.append(data)
        else:
            for v in data.values():
                l.extend(find_dicts_deep(v, fn_match))
    elif isinstance(data, list):
        for i in data:
            l.extend(find_dicts_deep(i, fn_match))
    return l

def find_all_cursors(data):
    return find_dicts_deep(
        data,
        lambda d: d.get('itemType') == 'TimelineTimelineCursor'
    )

def find_all_tweets(data):
    return find_dicts_deep(
        data,
        lambda d: d.get('__typename') == 'Tweet'
    )

def find_all_users(data):
    return find_dicts_deep(
        data,
        lambda d: d.get('__typename') == 'User'
    )

AUTH_BEARER = 'FCQLqLq1Nz0KMPgFONqHaJMgdVJqX4NAbXacUxwCmjMNIMlAkcD3%wTpYYxQvVHmbHczFr3hzl67pTHVAAAAAEgDOQFAAAAAAAAAAAAAAAAAAAAA reraeB'[::-1]

async def get_guest_id(session):
    # TODO: guest id moved to html response "gt=
    headers = {
        'User-Agent': USERAGENT,
        'Accept': '*/*',
        'Accept-Language': 'en-US,en;q=0.5',
        'Referer': 'https://twitter.com/',
        'authorization': AUTH_BEARER,
    }

    return (await session.post(
        f'https://api.twitter.com/1.1/guest/activate.json',
        headers=headers,
    )).json()['guest_token']

async def check_guest_id(guest_id):
    async with httpx.AsyncClient(http2=True) as session:
        session.headers['User-Agent'] = USERAGENT
        session.headers['x-guest-token'] = guest_id
        response = await session.post(
            'https://twitter.com/i/api/1.1/branch/init.json',
            headers={
                'x-guest-token': guest_id,
                'authorization': AUTH_BEARER,
            },
            json={},
        )
        if response.status_code == 200:
            return guest_id

async def load_tweet(session, tweet_id, cursor):
    headers = {
        'Accept': '*/*',
        'Accept-Language': 'en-US,en;q=0.5',
        'content-type': 'application/json',
        'authorization': AUTH_BEARER,
        'Referer': 'https://twitter.com/',
        'x-twitter-client-language': 'en',
        'x-twitter-active-user': 'yes',
        'Origin': 'https://twitter.com',
    }

    variables = {
        'focalTweetId': tweet_id,
        'with_rux_injections': False,
        'includePromotedContent': True,
        'withCommunity': True,
        'withQuickPromoteEligibilityTweetFields': True,
        'withBirdwatchNotes': False,
        'withSuperFollowsUserFields': True,
        'withDownvotePerspective': False,
        'withReactionsMetadata': False,
        'withReactionsPerspective': False,
        'withSuperFollowsTweetFields': True,
        'withVoice': True,
        'withV2Timeline': True,
    }

    if cursor:
        variables.update({
            'cursor': cursor,
            'referrer': 'tweet'
        })

    params = {
        'variables': json.dumps(variables, separators=(',', ':')),
        'features': json.dumps({
            'responsive_web_twitter_blue_verified_badge_is_enabled': True,
            'responsive_web_graphql_exclude_directive_enabled': False,
            'verified_phone_label_enabled': False,
            'responsive_web_graphql_timeline_navigation_enabled': True,
            'responsive_web_graphql_skip_user_profile_image_extensions_enabled': False,
            'tweetypie_unmention_optimization_enabled': True,
            'vibe_api_enabled': True,
            'responsive_web_edit_tweet_api_enabled': True,
            'graphql_is_translatable_rweb_tweet_is_translatable_enabled': True,
            'view_counts_everywhere_api_enabled': True,
            'longform_notetweets_consumption_enabled': True,
            'freedom_of_speech_not_reach_appeal_label_enabled': False,
            'standardized_nudges_misinfo': True,
            'tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled': False,
            'interactive_text_enabled': True,
            'responsive_web_text_conversations_enabled': False,
            'responsive_web_enhance_cards_enabled': False,
        }, separators=(',', ':')),
    }

    r = await session.get(
        'https://api.twitter.com/graphql/VaihYjIIeVg4gfvwMgQsUA/TweetDetail',
        params=params,
        headers=headers,
        timeout=20.0,
    )

    try:
        return r.json()
    except Exception:
        print('json decode error', r.status_code, r.content)
        raise

def get_tweet_ids_belonging_to_thread(tweets, thread_id):
    # should ignore quoted tweets in `tweets` list
    thread = set()
    thread.add(thread_id)

    while True:
        added = 0
        for t in tweets:
            should_add = (
                t['rest_id'] not in thread
                and t['legacy'].get('in_reply_to_status_id_str', -1) in thread
            )
            if not should_add:
                continue
            thread.add(t['rest_id'])
            added += 1

        if added == 0:
            break

    return thread

async def load_tree(pool: AioPool, thread_id):
    total_count = 0

    already_fetched = set()  # (tweet_id, cursor)
    already_fetched.add((thread_id, None))

    loaded_tweets = {}  # id -> tweet
    loaded_users = {}  # id -> user

    expected_replies_count = collections.Counter()  # tweet id -> count
    got_replies_count = collections.Counter()

    pool.put_task(thread_id, None)

    while True:
        async for (args, kwargs, response) in pool.results_iter():
            tweet_id, cursor = args
            total_count += 1

            print(
                f'Fetched {tweet_id!r} {"*" if cursor else ""} '
                f'({pool.responses_count} / {total_count})'
            )

            if 'errors' in response:
                print('Errors')
                print(response.get('errors'))
                # continue

            tweets = find_all_tweets(response)
            users = find_all_users(response)
            cursors = find_all_cursors(response)

            for t in tweets:
                tl = t['legacy']
                tid = t['rest_id']
                expected_replies_count[tid] = max(
                    expected_replies_count[tid], tl['reply_count'])

                if tid in loaded_tweets:
                    continue

                loaded_tweets[tid] = t
                if tl.get('in_reply_to_status_id_str'):
                    got_replies_count[tl['in_reply_to_status_id_str']] += 1

            for u in users:
                if 'rest_id' not in u:
                    print(u)
                    continue
                loaded_users[u['rest_id']] = u

            for c in cursors:
                if c['cursorType'] == 'Top':
                    continue

                f = (tweet_id, c['value'])
                if f not in already_fetched:
                    already_fetched.add(f)
                    pool.put_task(*f)

        # when no cursors are left to fetch,
        # check which tweets don't have all replies loaded

        thread = get_tweet_ids_belonging_to_thread(
            loaded_tweets.values(),
            thread_id
        )

        added = 0

        for tweet_id, expected in expected_replies_count.items():
            got = got_replies_count[tweet_id]
            if expected > got and tweet_id in thread:
                f = (tweet_id, None)
                if f not in already_fetched:
                    already_fetched.add(f)
                    pool.put_task(*f)
                    added += 1

        print()
        print('ADDED', added)
        print()

        if added == 0:
            break

    return {
        'thread_tweet_id': thread_id,
        'tweets': list(loaded_tweets.values()),
        'users': list(loaded_users.values()),
    }

async def worker_fn(worker_id, session, tweet_id, cursor):
    while True:
        print(f'[{worker_id}] fetching {tweet_id!r} {"*" if cursor else ""}')

        failed_requests = 0  # non-transport errors
        while failed_requests < 5:
            try:
                r = await load_tweet(session, tweet_id, cursor)
                print(f'[{worker_id}] done')
                return r
            except httpx.RequestError as e:
                print(f'could not load tweet (network error). Error: {e}')
            except asyncio.exceptions.CancelledError:
                raise
            except Exception:
                failed_requests += 1
                print('could not load tweet. Traceback:')
                traceback.print_exc()
            await asyncio.sleep(3)

        while True:
            await asyncio.sleep(10)
            try:
                session.headers.pop('x-guest-token')
                session.headers['x-guest-token'] = await get_guest_id(session)
                break
            except asyncio.exceptions.CancelledError:
                raise
            except Exception:
                print('could not refetch token. Traceback:')
                traceback.print_exc()

async def main():
    thread_id = sys.argv[1]

    num_workers = 8

    try:
        with open('saved_tokens.txt') as f:
            tokens = [i.strip() for i in f.readlines()]
    except FileNotFoundError:
        tokens = []

    tokens = await asyncio.gather(*[
        sure(lambda: check_guest_id(t), max_sleep=2)
        for t in tokens
    ])
    tokens = [i for i in tokens if i]

    need_tokens = num_workers - len(tokens)
    if need_tokens > 0:
        print(f'Getting {need_tokens} tokens')
        new_tokens = await asyncio.gather(*[
            sure(lambda: get_guest_id(httpx.AsyncClient(http2=True)))
            for _ in range(need_tokens)
        ])
        tokens.extend(new_tokens)

    with open('saved_tokens.txt', 'w') as f:
        for i in tokens:
            f.write(f'{i}\n')

    worker_args = []
    for worker_id, t in enumerate(tokens):
        session = httpx.AsyncClient(http2=True)
        session.headers['User-Agent'] = USERAGENT
        session.headers['x-guest-token'] = t

        worker_args.append((worker_id, session))

    pool = AioPool(num_workers, worker_fn, worker_args)

    st = time.monotonic()
    thread = await load_tree(pool, thread_id)

    print()
    print('*' * 80)
    print('DONE', pool.requests_count, pool.responses_count)
    print('TOOK', time.monotonic() - st)
    print()

    await pool.shutdown()

    with open(f'thread_{thread_id}.json', 'w') as f:
        json.dump(thread, f)

if __name__ == '__main__':
    # asyncio.run(main(), debug=True)
    # exit(0)
    if uvloop_installed:
        if sys.version_info >= (3, 11):
            with asyncio.Runner(loop_factory=uvloop.new_event_loop) as runner:
                runner.run(main())
        else:
            uvloop.install()
            asyncio.run(main())
    else:
        asyncio.run(main())


#!/usr/bin/env python3

def add_gstreamer_packages():
    import os
    import sys
    from distutils.sysconfig import get_python_lib

    dest_dir = get_python_lib()

    packages = ['gobject', 'glib', 'pygst', 'pygst.pyc', 'pygst.pth',
                'gst-0.10', 'pygtk.pth', 'pygtk.py', 'pygtk.pyc']

    python_version = sys.version[:3]
    global_path = os.path.join('/usr/lib', 'python' + python_version)
    global_sitepackages = [os.path.join(global_path,
                                        'dist-packages'),  # for Debian-based
                           os.path.join(global_path,
                                        'site-packages')]  # for others

    for package in packages:
        for pack_dir in global_sitepackages:
            src = os.path.join(pack_dir, package)
            dest = os.path.join(dest_dir, package)
            if not os.path.exists(dest) and os.path.exists(src):
                os.symlink(src, dest)

def check_gstreamer():
    try:
        import gobject, pygst
    except ImportError:
        add_gstreamer_packages()

if __name__ == '__main__':
    check_gstreamer()


from kivy.app import App
from kivy.uix.label import Label
from kivy.uix.accordion import Accordion,AccordionItem
from kivy.uix.button import Button
class MyApp(App):
    def build(self):
        a1=Accordion()
        a1.orientation='vertical'
        for x in range(5):
            i=AccordionItem(title='Title %d'%x)
            i.add_widget(Label(text='Label %i' % x))

            a1.add_widget(i)

        return a1

if __name__ == '__main__':
    MyApp().run()