from django.contrib.auth.decorators import login_required, user_passes_test
from django.shortcuts import render, redirect
from django.http import JsonResponse
from django.db.models import Case, When, Sum, F, Min, Value, CharField, BooleanField, Q
import json, random
from index.defs import get_array, execute_query
from inventory.models import Inventory
from index.models import Player
from recon.models import Animal, Hunt
from map.models import Hexa
from inventory.views import make_grid

craft_box_range = [21, 26]

@login_required(redirect_field_name=None)
def index(request):

    player, hexa, animals, situation_instance = entry(request)

    state = ['', 'attacking', 'running', 'dead']

    animals_with_state = []
    for animal in animals:
        print(animal)
        animals_with_state.append(animal + (state[animal[3]],) )

    grid = make_grid(request, grid_max=34)
    slices = [6, 11, 16, 21, 24, 27, 29, 31, 33]

    if not hexa[5]:
        pass
    else:
        pass

    return render(request, 'recon.html', {'animals': animals_with_state,
                                          'grid': grid,
                                          'slices': slices,
                                          'craft_box_start': craft_box_range[0],
                                          'animals_start': craft_box_range[1]+1,
                                          })

def recon(request):
    if request.is_ajax():
        if request.method == 'POST':
            player, hexa, situation_list, situation = entry(request)
            animals, status = move(player, hexa, situation)
            return JsonResponse([animals, status], safe=False)

def leave_button(request):
    if request.is_ajax():
        if request.method == 'POST':
            animals, status, remaining = leave(request)
            return JsonResponse([animals, status], safe=False)

def leave(request):
    player, hexa, situation_list, situation = entry(request)
    attacking = situation.filter(behaviour=1)
    remaining = False
    for attacker in attacking:
        if random.randint(0, 100) < 75:
            attacker.delete()
        else:
            remaining = True

    animals, status = move(player, hexa, situation, leaving=True)
    return animals, status, remaining

def shot(request):
    if request.is_ajax():
        if request.method == 'POST':
            animal_number = json.loads(request.body.decode('utf-8'))
            hunt = Hunt.objects.filter(id=animal_number, player_id=request.user.active_player)

            animal = hunt.select_related('animal')\
                .values_list('hit_chance', 'hits', 'animal__shots', 'animal__aggression', 'behaviour',
                             'animal__meat', 'animal__leather', 'animal__feather', 'animal__scales', 'animal__venom')[0]

            Hunt.objects.filter(player_id=request.user.active_player, behaviour=2).exclude(id=animal_number).delete()

            status = 'miss'

            player, hexa, situation_list, situation = entry(request)

            if animal[0] >= random.randint(0, 100):
                status = 'hit'
                if int(animal[1])+1 >= animal[2]:
                    hunt.update(behaviour=3)
                    status = 'dead'
                    pos = craft_box_range[1]+1+(hunt[0].slot*2)
                    second = False
                    for i in range(1, 6):
                        if animal[i+4]:
                            if second: pos += 1
                            Inventory(player=player[0], pos=pos, consume_items_id=i, amount=animal[i+4]).save()
                            second = True
                else:
                    run_chance = random.randint(0, 14)*10+random.randint(0, 10)
                    if animal[4] != 1 and run_chance > animal[3]:
                        hunt.update(behaviour=2, hits=F('hits') + 1)
                    else:
                        hunt.update(behaviour=1, hits=F('hits') + 1)
            else:
                if animal[4]:
                    hunt.exclude(behaviour=1 or 3).delete()

            animals = on_place(player, hexa, situation)

            return JsonResponse([animals, status], safe=False)

def drag_and_drop(request):
    if request.is_ajax():
        if request.method == 'POST':
            arr = json.loads(request.body.decode('utf-8'))
            execute_query(
                query="""UPDATE inventory_inventory SET pos = %s
                         WHERE player_id = %s AND id = %s""",
                params=[arr["entNumber"], request.user.active_player, arr["dragNumber"]])

            animals = []

            if arr['initNumber'] > craft_box_range[1]:
                animals = loot(request)

            return JsonResponse(animals, safe=False)

def loot(request):
    player, hexa, situation_list, situation = entry(request)
    animals = on_place(player, hexa, situation)
    inventory = Inventory.objects.filter(player=request.user.active_player, pos__gt=craft_box_range[1]).values_list('pos')

    status = 'looting'

    for dead in situation.filter(behaviour=3):
        if dead.slot+craft_box_range[1] not in inventory:
            status += ' | '+str(dead.slot)

    return [animals, status]

def entry(request):
    player = Player.objects.filter(pk=request.user.active_player)
    hexa = player.select_related('pos') \
        .values_list('pos__grass', 'pos__wood', 'pos__hills', 'pos__mountains', 'pos__water', 'pos__town')[0]

    print('grass %d, wood %d, hills %d, mountains %d, water %d' % (hexa[0], hexa[1], hexa[2], hexa[3], hexa[4]))

    if not hexa[5]:
        situation_instance = Hunt.objects.filter(player=request.user.active_player).select_related('animal')
        situation = list(situation_instance.values_list('id', 'animal__name', 'hit_chance', 'behaviour', 'slot'))
    else:
        situation = None
        situation_instance = None

    return player, hexa, situation, situation_instance

def move(player, hexa, situation, leaving=False):
    animals = []

    attacking = situation.filter(behaviour=1).select_related('animal')\
        .values_list('id', 'animal__name', 'animal__size', 'hits', 'slot')

    for attacker in attacking:
        hit_chance = attacker[2] - random.randint(0, 30 - 12 * attacker[3])
        animals.append([attacker[0], attacker[1], hit_chance, 1, attacker[4]])

    if situation:
        situation.exclude(behaviour=1).delete()

    Inventory.objects.filter(player=player[0], pos__gt=craft_box_range[1]).delete()

    status = ''

    if not leaving:
        for s in range(4):
            if situation.filter(slot=s):
                continue
            animals, status = spawn_animal(animals, hexa, player, s)

    return animals, status

def on_place(player, hexa, situation):
    animals = []

    no_idles = situation.exclude(behaviour=0).select_related('animal')\
        .values_list('id', 'animal__name', 'animal__size', 'hits', 'behaviour', 'slot')

    for no_idle in no_idles:
        hit_chance = no_idle[2] - random.randint(0, 30) + 12 * no_idle[3]
        animals.append([no_idle[0], no_idle[1], hit_chance, no_idle[4], no_idle[5]])

    if situation:
        situation.filter(behaviour=0).delete()

    startled = random.randint(1, 3)
    print('startled %d' % startled)

    for s in range(4):
        if situation.filter(slot=s):
            continue
        if startled > 0:
            startled -= 1
            continue
        animals = spawn_animal(animals, hexa, player, s)
    return animals

def spawn_animal(animals, hexa, player, s):
    aim = random.randint(0, 660)
    biome = 0
    status = ''
    for i in range(5):
        biome += hexa[i]
        if aim < biome:
            shot = random.randint(0, 100)
            animal_instance = Animal.objects.filter(biome=i, presence__gte=shot).order_by('presence')
            animal = animal_instance.values_list('id', 'name', 'aggression', 'shots', 'size').first()
            if animal:
                if animal[0] < 12:
                    attack_chance = random.randint(0, 9) * 10 + random.randint(0, 10)
                    behaviour = 1 if attack_chance <= animal[2] else 0
                    hit_chance = animal[4] - random.randint(0, 18 if behaviour == 1 else 30)
                else:
                    behaviour = 3
                    hit_chance = 0

                    pos = craft_box_range[1] + 1 + s * 2
                    Inventory(player=player[0], pos=pos, consume_items_id=7, amount=random.randint(1, 3)).save()
                    status = 'items'

                hunt = Hunt(player=player[0], animal=animal_instance[0], behaviour=behaviour,
                            hit_chance=hit_chance, slot=s)
                hunt.save()
                animals.append([hunt.pk, animal[1], hit_chance, behaviour, s])
            break
    return animals, status


import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np
from typing import List, Tuple
from kfold import KFold
from concept import mobility, mobility_diff, frontier, frontier_diff, steady, corner, corner_steady, steady_diff, corner_diff, score_f1, score
from read_wth import read_wthor_files
import pandas as pd
from player_color import BLACK, WHITE, EMPTY
from device_detect import detectDevice
from load_net import loadNetOutput
from load_data import into_input_format_2, load_tensor
import time
import random
from rewrite_print import print, print_acc
import warnings
warnings.filterwarnings("ignore")  # 忽略UserWarning兼容性警告

batchSize = 128

class LinearModel(nn.Module):  # 定义线性模型g,用于c-con
    def __init__(self, input_size):
        super(LinearModel, self).__init__()
        self.linear = nn.Linear(input_size, 1)

    def forward(self, x: torch.Tensor):
        x = x.view(x.size(0), -1)
        y = self.linear(x)
        return y.squeeze()

def batch_normalize(x, gamma=0.5, beta=0.5, eps=1e-5):
    # N, D = x.shape
    mean = np.mean(x)
    var = np.var(x)
    x_hat = (x - mean) / np.sqrt(var + eps)
    out = gamma * x_hat + beta
    return out

def loss_f1(pred: torch.Tensor, target: torch.Tensor, w, lamb, device: torch.device) -> torch.Tensor:
    mse_loss = torch.mean((pred - target)**2)
    # l2_loss = torch.norm(w)/pred.size(0)
    l1_loss = torch.sum(torch.abs(w))/pred.size(0)
    return mse_loss + lamb * l1_loss  # 返回的是平均loss

# 定义训练函数
def train(model: LinearModel, X_train: torch.Tensor, y_train: torch.Tensor, lamb: float, device: torch.device, learning_rate, num_epochs=500):
    optimizer = torch.optim.SGD(
        model.parameters(), lr=learning_rate, momentum=0.9)
    schduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
        optimizer, mode='min', patience=30, factor=0.7)

    X_train = load_tensor(X_train.clone().detach(), batchSize)
    y_train = load_tensor(y_train.reshape(-1), batchSize)

    for epoch in range(num_epochs):
        total_loss = 0
        datasize = 0
        for i in range(len(X_train)):

            inputs = X_train[i]
            targets = y_train[i]
            datasize += targets.size(0)
            optimizer.zero_grad()
            outputs = model(inputs)

            loss = loss_f1(outputs, targets, model.linear.weight, lamb, device)
            total_loss += loss.item()*targets.size(0)
            loss.backward()
            optimizer.step()
            schduler.step(loss)

        if num_epochs > 500 and (epoch+1) % 500 == 0:
            print('LR: {}, Epoch [{}/{}], Loss: {:.4f}'.format(
                learning_rate, epoch + 1, num_epochs, total_loss/datasize))

# 正则化参数通过交叉验证确定

def decide_lambda_lr(z_d: torch.Tensor, c_z: torch.Tensor, n_splits: int, device: torch.device, random_seed: int = 42):  # z_d为probe的输入，c_z为标签数据
    kf = KFold(n_splits, shuffle=True, random_state=random_seed)
    lamb_list = [0.01, 0.1, 1, 10]  # 网格化搜索之后发现lamb的影响很小，为节省时间干脆都定为10
    lr_list = [1e-06, 1e-07, 1e-08, 1e-09]
    best_loss_lamb = float('inf')
    best_lamb = None
    best_lamb_lr = None

    for lamb in lamb_list:  # 决定了best_lamb及其下的best_loss和best_lr
        best_lr = None
        best_loss = float('inf')
        for lr in lr_list:  # 决定了当前lamb下的best_loss和best_lr
            total_loss = 0
            datasize = 0
            for k, index in enumerate(kf.split(z_d)):
                train_index, val_index = index
                X_train, X_val = z_d[train_index], z_d[val_index]
                y_train, y_val = c_z[train_index], c_z[val_index]
                # 定义模型并训练
                input_size = z_d.shape[1]*z_d.shape[2]*z_d.shape[3]
                linear_model = LinearModel(input_size)
                linear_model = linear_model.to(device)
                # print(f'-------------The {k+1}/{n_splits} th training------------')
                train(linear_model, X_train.reshape(-1, input_size),
                      y_train.reshape(-1, 1), lamb, device, learning_rate=lr, num_epochs=500)

                # 验证
                inputs = load_tensor(X_val)
                targets = torch.tensor(y_val, device=device)
                batch_output = linear_model(inputs[0])
                for i in range(1, len(inputs)):
                    batch_output = torch.cat(
                        [batch_output, linear_model(inputs[i])], dim=0)

                loss = loss_f1(batch_output, targets,
                               linear_model.linear.weight, lamb, device)
                total_loss += loss.item()*targets.size(0)
                datasize += targets.size(0)

            avg_loss = total_loss / datasize

            if avg_loss < best_loss:
                best_loss = avg_loss
                best_lr = lr

        print(f"lambda: {lamb}, best lr: {best_lr}, best loss: {best_loss}")

        if best_loss < best_loss_lamb:
            best_loss_lamb = best_loss
            best_lamb = lamb
            best_lamb_lr = best_lr

    print(
        f"best lambda: {best_lamb}, best lr: {best_lamb_lr}, best loss: {best_loss_lamb}")
    return best_lamb, best_lamb_lr

def concept_probe(concept_name: str, train_val_data: pd.DataFrame, train_zd: torch.Tensor, device: torch.device, num_epochs: int = 1000):
    y_train = []
    for i in range(len(train_val_data)):
        board = train_val_data.iloc[i, :]
        c_z = concept_name(board, BLACK)
        y_train.append(c_z)

    y_train = torch.tensor(y_train, device=device)

    best_lamb, best_lr = decide_lambda_lr(train_zd, y_train, 3, device)
    input_size = train_zd.shape[1]*train_zd.shape[2]*train_zd.shape[3]

    linear_model = LinearModel(input_size)
    linear_model = linear_model.to(device)
    train(linear_model, train_zd.reshape(-1, input_size),
          y_train.reshape(-1, 1), best_lamb, device, best_lr, num_epochs)
    return linear_model

def r2_score_0(y_test, X_test):
    SStot = np.sum((y_test-np.mean(y_test))**2)
    SSres = np.sum((y_test-X_test)**2)
    r2 = 1-SSres/SStot
    return r2

def r2_score(y_test, X_test):
    r = np.corrcoef(X_test, y_test)[0, 1]
    return r**2

def test_accuracy(concept_name: str, model: LinearModel, test_data: pd.DataFrame, test_zd: torch.Tensor):
    y_test = []
    for i in range(len(test_data)):
        board = test_data.iloc[i, :]
        c_z = concept_name(board, BLACK)
        y_test.append(c_z)

    test_zd = load_tensor(test_zd)
    X_test = model(test_zd[0])
    for i in range(1, len(test_zd)):
        X_test = torch.cat([X_test, model(test_zd[i])], dim=0)

    r_squared = r2_score(np.array(y_test), np.array(X_test.tolist()))
    r_squared_0 = r2_score_0(np.array(y_test), np.array(X_test.tolist()))
    print(f"r^2: {r_squared}, r^2_0: {r_squared_0}")
    print("---------------------------------------------------------------\n")
    return r_squared, r_squared_0

# 这个部分只算一次，之后从结果中拿取即可
def zdlist(device: torch.device, epoch_list: List[int], trainset, testset):
    all_trainzd_list = []
    all_testzd_list = []
    for training_step in epoch_list:
        train_z_d_list = loadNetOutput(
            training_step, into_input_format_2(trainset), device)
        test_z_d_list = loadNetOutput(
            training_step, into_input_format_2(testset), device)
        all_trainzd_list.append(train_z_d_list)
        all_testzd_list.append(test_z_d_list)
    return all_trainzd_list, all_testzd_list  # list的长度就等于epoch_list的长度

def compute_results(device: torch.device, training_step, concept, trainset, testset, train_z_d_list, test_z_d_list, layer_num=15):
    accuracy_layer = []
    accuracy_layer_0 = []

    for layer in range(layer_num):
        print(
            f"---------NN training epoch: {training_step}, Layer num: {layer+1}/15---------")
        linear_model = concept_probe(
            concept, trainset, train_z_d_list[layer], device)
        accuracy, accuracy_0 = test_accuracy(
            concept, linear_model, testset, test_z_d_list[layer])

        accuracy_layer.append(accuracy)  # 记录当前training_step下遍历各layers的accuracy
        accuracy_layer_0.append(accuracy_0)

    return accuracy_layer, accuracy_layer_0

if __name__ == '__main__':

    globalDevice = detectDevice()
    BLACK = -1

    file_list = range(2000, 2022)  # 代表数据年份，可选：range(1977,2024)
    epoch_list = range(1, 58)  # 代表神经网络训练代数，可选：range(1,58)
    concept_list = [frontier, steady, corner, mobility, frontier_diff,
                    score_f1, score, steady_diff,  corner_diff, mobility_diff]

    paths = ['./gamedata/WTH_' + str(i)+'.wtb' for i in file_list]
    trainset, testset = read_wthor_files(paths)

    all_trainzd_list, all_testzd_list = zdlist(
        globalDevice, epoch_list, trainset, testset)

    for concept in concept_list:
        print(
            f'Current time: {time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())}')
        print(
            f'Use datasets in years: {file_list[0]}~{file_list[-1]}. Size of trainset: {len(trainset)}')
        print(f'Concept: {str(concept.__name__)}\n')

        accuracy_step, accuracy_step_0 = [], []
        for training_step in epoch_list:
            train_z_d_list = all_trainzd_list[training_step]
            test_z_d_list = all_testzd_list[training_step]
            accuracy_layer, accuracy_layer_0 = compute_results(
                globalDevice, training_step, concept, trainset, testset, train_z_d_list, test_z_d_list)
            accuracy_step.append(accuracy_layer)
            accuracy_step_0.append(accuracy_layer_0)

        print_acc(f"concept: {str(concept.__name__)}")
        print_acc(f"accuracy_step: {accuracy_step}")
        print_acc(f"accuracy_step_0: {accuracy_step_0}")
        print_acc(
            '---------------------------------------------------------------\n')


from __future__ import print_function
import os
import sys
import argparse

# our tools are in "libexec"
sys.path.append(os.path.join(sys.path[0], "libexec"))

import kdz

class KDZFileTools(kdz.KDZFile):
	"""
	LGE KDZ File creation tool
	"""

	indir = "kdzextracted"

#kdz.KDZFile._dz_header

	def loadParams(self):
		"""
		Loads the .kdz.params file for output, saves the values
		"""

		params = dict()
		file = open(os.path.join(self.indir, ".kdz.params"), "rt")
		line = file.readline()
		while len(line) > 0:
			line.lstrip()
			line = line.partition("#")[0]
			if len(line) == 0:
				line = file.readline()
				continue
			parts = line.partition("=")
			if len(parts[1]) == 0:
				print("[!] Bad line in {:s}'s parameter file".format(self.indir), file=sys.stderr)
			var = parts[0].rstrip()
			try:
				val = int(parts[2].strip())
			except ValueError:
				val = parts[2].strip()
			params[var] = val
			line = file.readline()

		file.close()

		for k in 'version', 'dataStart', 'payload0', 'payload0head':
			if k not in params:
				print("Parameter value \"{:s}\" is missing, unable to continue".format(k))
				sys.exit(1)

		if params['version'] != 2:
			print("[!] File format version not 2, cannot continue", file=sys.stderr)
			sys.exit(1)

		self.dataStart = params['dataStart']

		i = 0
		self.headers = []
		headers = {}
		self.payload = []
		self.files = {}
		name = "payload"+str(i)
		while name in params:
			self.payload.append(params[name])
			headers[params[name+"head"]] = params[name]
			i += 1
			name = "payload"+str(i)
		for i in range(i):
			self.headers.append(headers[i])

	def cmdCreateFile(self):
		"""
		Create the specified KDZ file
		"""

		out = open(self.kdzfile, "wb")
		current = self.dataStart
		out.seek(current, os.SEEK_SET)

		for name in self.payload:
			print("[+] Writing {:s} to output file {:s}".format(name, self.kdzfile))
			inf = open(os.path.join(self.indir, name), "rb")
			buf = " "
			while len(buf) > 0:
				buf = inf.read(4096)
				out.write(buf)
			self.files[name] = [current, out.tell() - current]
			inf.close()
			current = out.tell()

		self.files[self.headers[-1]].append(0)

		print("\n[+] Writing headers to {:s}".format(self.kdzfile))

		out.seek(0, os.SEEK_SET)
		out.write(self._dz_header)
		for name in self.headers:
			# last record
			if len(self.files[name]) > 2:
				out.write(b'\x03')
			head = {
				'name':		name.encode("utf8"),
				'length':	self.files[name][1],
				'offset': 	self.files[name][0],
			}
			buf = self.packdict(head)
			out.write(buf)
		out.close()
		print("[+] Done!")

	def cmdList(self):
		pass

	def parseArgs(self):
		# Parse arguments
		parser = argparse.ArgumentParser(description='LG KDZ File creator by Elliott Mitchell')
		parser.add_argument('-f', '--file', help='KDZ File to read', action='store', required=True, dest='kdzfile')
		group = parser.add_mutually_exclusive_group(required=True)
		group.add_argument('-l', '--list', help='list partitions', action='store_true', dest='listOnly')
		group.add_argument('-m', '--make', help='extract all partitions', action='store_true', dest='createFile')
		parser.add_argument('-d', '--dir', help='input directory', action='store', dest='indir')

		return parser.parse_args()

	def main(self):
		args = self.parseArgs()
		self.kdzfile = args.kdzfile

		if args.indir:
			self.indir = args.indir

		self.loadParams()

		if args.listOnly:
			self.cmdList()

		elif args.createFile:
			self.cmdCreateFile()

if __name__ == "__main__":
	kdztools = KDZFileTools()
	kdztools.main()


import streamlit as st
import pandas as pd
import pyarrow.parquet as pq

from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score

header         = st.container()
dataset        = st.container()
features       = st.container()
model_training = st.container()

st.markdown(
    """
    <style>
    .main {
    background-color: #F5F5F5;
    }
    </style>
    """,
    unsafe_allow_html=True
)

@st.cache_data
def get_data(filename):
    taxi_data = pq.read_table(filename)
    taxi_data = taxi_data.to_pandas()
    return taxi_data

with header:
    st.title('Welcome to my awesome data science project!')

with dataset:
    st.header('NYC taxi dataset')
    st.text("I found this dataset at: https://www.nyc.gov/site/tlc/about/tlc-trip-record-data.page")

    taxi_data = get_data('Data/taxi_data.parquet')
    taxi_data = taxi_data.head(100000)
    st.write(taxi_data.head())

    st.subheader('Pick-up location ID distribution on the NYC dataset')
    pulocation_dist = pd.DataFrame(taxi_data['PULocationID'].value_counts()).head(50)
    st.bar_chart(pulocation_dist)

with features:
    st.header('The features I created')

    st.markdown('* **first feature:** I created this feature because of this... I calculated it using this log...')
    st.markdown('* **second feature:** I created this feature because of this... I calculated it using this log...')

with model_training:
    st.header('Time to train the model!')
    st.text("Here you get to choose the hyperparameters of the model and see how the performance changes.")

    sel_col, disp_col = st.columns(2)

    max_depth = sel_col.slider('Whats should the max_depth of the model be?', min_value=10, max_value=100, value=20, step=10)

    n_estimators = sel_col.selectbox('How many trees should there be?', options=[100,200,300,'No limit'], index=0)

    input_feature = sel_col.selectbox('Which feature should be used as the input feature', taxi_data.keys())

    if n_estimators == 'No limit':
        regr = RandomForestRegressor(max_depth=max_depth)
    else:
        regr = RandomForestRegressor(max_depth=max_depth, n_estimators=n_estimators)

    x = taxi_data[[input_feature]].values
    y = taxi_data[['trip_distance']].values

    regr.fit(x,y)
    prediction = regr.predict(y)

    disp_col.subheader('Mean absolute error of the model is: ')
    disp_col.write(mean_absolute_error(y, prediction))

    disp_col.subheader('Mean squared error of the model is:')
    disp_col.write(mean_squared_error(y, prediction))

    disp_col.subheader('R squared score of the model is:')
    disp_col.write(r2_score(y, prediction))


import heapq

def topk(a,k):
    result = []

    for i in range(0,k):
        heapq.heappush(result, a[i])

    for i in range(k, len(a)):
        if a[i] > result[0]:
            heapq.heappop(result)
            heapq.heappush(result, a[i])

    return result

if __name__ == "__main__":
    #a = [1,3,2,6,-1,4,1,8,2]

    a = [3,1,5,12,2,11]
    k = 3
    #k=5
    result = topk(a,k)

    print(result)


from lxml import etree
import unicodedata
from pathlib import Path
from progressbar import progressbar
import json

config = json.load(open("../conf/config.json"))

download_dir = Path("../build/download")
parsed_dir = Path("../build/parsed")
normalize_char_map = {'ё':'е', 'Ё':'Е'}
vowels = 'аэыуояеёюи'

def normalize_string(s):
   norms = unicodedata.normalize('NFC', s)
   noacc =        [c
                  for c in norms
                  if unicodedata.category(c) != 'Mn' #'Mark, Nonspacing' = accents
                  and (
                      not unicodedata.category(c).startswith('P') #Punctuation
                      or c == '-'
                  )]
   normalized = ''.join(normalize_char_map[c] if c in normalize_char_map else c
                  for c in noacc).lower()
   stress=None
   if vowel_count(normalized) > 1:
       for p, c in enumerate(norms):
           if unicodedata.category(c) == 'Mn':  # 'Mark, Nonspacing' = accents
               stress = p
               break

   return tuple([normalized, stress, ''.join(noacc)])

def add_norm(forms, html, xpath, optional_prefix=None, prefix_norm=None):
    matches = html.xpath(xpath)
    for match in matches:
        match_text = match.xpath("string(.)")

        #Cell may contain multiple forms, e.g. свой -> свое́й, свое́ю (https://en.wiktionary.org/wiki/%D1%81%D0%B2%D0%BE%D0%B9)
        for match_form in match_text.split(", "):

            # Add comparative with and without comparative suffix, e.g. попроще and проще
            if optional_prefix and match_form.startswith(optional_prefix):
                suffix = match_form[len(optional_prefix):]
                forms.add(normalize_string(prefix_norm + suffix))
                forms.add(normalize_string(suffix))
            else:
                forms.add(normalize_string(match_form))

def vowel_count(txt):
    count = 0
    txt = txt.lower()
    for vowel in vowels:
        count = count + txt.count(vowel)
    return count

def parse_file(f, src_lang, destdir):

    pageJson=json.load(open(str(f)))
    of=pageJson['html']
    title = pageJson['title']
    html = etree.fromstring(of)

    for target_lang, langpair in config["langpairs"][src_lang].items():
        lang_name = langpair["lang_span_name"]
        langs = html.xpath("//h2/span[text()='%s' and contains(@class,'mw-headline')]" % lang_name)
        if not langs: #does not work for Serbo-Croatian
            continue
        forms = set()

        span_selector = "//*[preceding-sibling::h2[1]/span[text()='%s']]" % lang_name
        td_selector_template = "%s//table[contains(@class,'inflection-table')]/%s/tr/td//span[@lang='%s']"
        for tbody_selector in ['tbody', '.']:
            td_selector = td_selector_template % (span_selector, tbody_selector, target_lang)
            add_norm(forms, html, td_selector)
        add_norm(forms, html, "%s//strong[contains(@class,'headword') and @lang='%s']" % (span_selector, target_lang))

        # Parse comparative. NB тёплый has two variants
        comp_select = "//b[@lang='%s' and preceding-sibling::*[name()='i' and text()='comparative']]" % target_lang
        add_norm(forms, html, comp_select, "(по)", "по")

        dir = Path(destdir, target_lang)
        dir.mkdir(parents=True, exist_ok=True)
        file = Path(dir, Path(f).with_suffix('.dat').name)
        s = ''.join(["%s\t%s\t%s\t%s\n" % (form[0], title, form[1] if form[1] else 0, form[2]) for form in forms])
        file.write_text(s, encoding='utf8')

        marker.write_bytes(b'')

for src_lang, targets in config["langpairs"].items():
    lang_dir = Path(download_dir, src_lang)

    destdir = Path(parsed_dir, src_lang)
    marker_dir = Path(destdir, "_done")
    marker_dir.mkdir(parents=True, exist_ok=True)

    print("Source language: [%s]" % src_lang)
    print("Listing files...")
    files = sorted(lang_dir.glob("*.json"))

    new_pages = 0

    print("Parsing files...")
    for f in progressbar(files):

        marker = Path(marker_dir, Path(f).name)
        if marker.is_file():
            continue

        new_pages = new_pages + 1

        parse_file(f, src_lang, destdir)

    print("Parsed %d new pages out of %d total pages." % (new_pages, len(files)))


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import datetime
from django.utils.timezone import utc

class Migration(migrations.Migration):

    dependencies = [
        ('bar', '0031_auto_20151210_1746'),
        ('personal', '0051_auto_20151210_1657'),
    ]

    operations = [
        migrations.CreateModel(
            name='EmpleadoTelefono',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('telefono', models.IntegerField(help_text=b'Ingrese el telefono fijo o movil del cliente. El dato debe contener solo numeros.')),
                ('codigo_ciudad_operadora_telefono', models.ForeignKey(default=21, to='bar.CodigoCiudadOperadoraTelefono')),
                ('codigo_pais_telefono', models.ForeignKey(default=595, to='bar.CodigoPaisTelefono')),
            ],
        ),
        migrations.RemoveField(
            model_name='empleado',
            name='telefono',
        ),
        migrations.RemoveField(
            model_name='empleado',
            name='telefono_movil',
        ),
        migrations.AddField(
            model_name='empleado',
            name='sexo',
            field=models.CharField(default=b'F', max_length=1, choices=[(b'F', b'Femenino'), (b'M', b'Masculino')]),
        ),
        migrations.AlterField(
            model_name='empleado',
            name='fecha_nacimiento',
            field=models.DateField(default=datetime.datetime(2015, 12, 10, 20, 46, 43, 529000, tzinfo=utc)),
        ),
        migrations.AlterField(
            model_name='horario',
            name='horario_fin',
            field=models.TimeField(default=datetime.datetime(2015, 12, 10, 20, 46, 43, 539000, tzinfo=utc)),
        ),
        migrations.AlterField(
            model_name='horario',
            name='horario_inicio',
            field=models.TimeField(default=datetime.datetime(2015, 12, 10, 20, 46, 43, 538000, tzinfo=utc)),
        ),
        migrations.AddField(
            model_name='empleadotelefono',
            name='empleado',
            field=models.ForeignKey(to='personal.Empleado'),
        ),
    ]


import tncontract as tn
import numpy as np

def con(*args):
    """
    Contract a network of tensors. Similar purpose to NCON, described in
    arxiv.org/abs/1402.0939, but designed to work with the Tensor objects of
    tncontract.

    Examples
    --------

    >>> import tncontract as tn

    For the examples below, we define three tensors

    >>> A = tn.Tensor(np.random.rand(3,2,4), labels=["a", "b", "c"])
    >>> B = tn.Tensor(np.random.rand(3,4), labels=["d", "e"])
    >>> C = tn.Tensor(np.random.rand(5,5,2), labels=["f", "g", "h"])

    Contract a pair indices between two tensors
    -------------------------------------------
    The following contracts  pairs of indices "a","d" and "c","e" of tensors
    `A` and `B`. It is identical to A["a", "c"]*B["d", "e"]

    >>> tn.con(A, B, ("a", "d" ), ("c", "e"))
    Tensor object: shape = (2), labels = ["b"]

    Contract a pair of indices beloning to one tensor (internal edges)
    ------------------------------------------------------------------
    The following contracts the "f" and "g" indices of tensor `C`

    >>> t.con(C, ("f", "g"))
    Tensor object: shape = (2), labels = ["h"]

    Return the tensor product of a pair of tensors
    ----------------------------------------------
    After all indices have been contracted, `con` will return the tensor
    product of the disconnected components of the tensor contraction. The
    following example returns the tensor product of `A` and `B`.

    >>> tn.con(A, B)
    Tensor object: shape = (3, 2, 4, 3, 4), labels = ["a", "b", "c", "d", "e"]

    Contract a network of several tensors
    -------------------------------------
    It is possible to contract a network of several tensors. Internal edges are
    contracted first then edges connecting separate tensors, and then the
    tensor product is taken of the disconnected components resulting from the
    contraction. Edges between separate tensors are contracted in the order
    they appear in the argument list. The result of the example below is a
    scalar (since all indices will be contracted).

    >>> tn.con(A, B, C, ("a", "d" ), ("c", "e"), ("f", "g"), ("h", "b"))

    Notes
    -----
    Lists of tensors and index pairs for contraction may be used as arguments.
    The following example contracts 100 rank 2 tensors in a ring with periodic
    boundary conditions.
    >>> N=100
    >>> A = tn.Tensor(np.random.rand(2,2), labels=["left","right"])
    >>> tensor_list = [A.suf(str(i)) for i in range(N)]
    >>> idx_pairs = [("right"+str(j), "left"+str(j+1)) for j in range(N-1)]
    >>> tn.con(tensor_list, idx_pairs, ("right"+str(N-1), "left0"))
    """

    tensor_list = []
    contract_list = []
    for x in args:
        #Can take lists of tensors/contraction pairs as arguments
        if isinstance(x, list):
            if isinstance(x[0], tn.Tensor):
                tensor_list.extend(x)
            else:
                contract_list.extend(x)

        elif isinstance(x, tn.Tensor):
            tensor_list.append(x)
        else:
            contract_list.append(x)

    tensor_list = [t.copy() for t in tensor_list] #Unlink from memory
    all_tensor_indices = [t.labels for t in tensor_list]

    #Check that all no index is specified in more than one contraction
    contracted_indices = [item for pair in contract_list for item in pair]
    if len(set(contracted_indices)) != len(contracted_indices):
        raise ValueError("Index found in more than one contraction pair.")

    index_lookup = {}
    for i,labels in enumerate(all_tensor_indices):
        for lab in labels:
            if lab in index_lookup.keys():
                raise ValueError("Index label "+lab+" found in two tensors."+
                        " Tensors must have unique index labelling.")
            index_lookup[lab] = i

    internal_contract = [] #Indicies contracted within the same tensor
    pairwise_contract = [] #Indicies contracted between different tensors
    tensor_pairs = []
    tensors_involved = set()
    for c in contract_list:
        if index_lookup[c[0]] == index_lookup[c[1]]:
            internal_contract.append(c)
        else:
            #Takes into account case where multiple indices from a pair of
            #tensors are contracted (will contract in one call to np.dot)
            #TODO: Better to flatten first?
            if (tuple(np.sort((index_lookup[c[0]],index_lookup[c[1]])))
                    in tensor_pairs):
                idx = tensor_pairs.index((index_lookup[c[0]],
                    index_lookup[c[1]]))
                if not isinstance(pairwise_contract[idx][0], list):
                    pairwise_contract[idx][0] = [pairwise_contract[idx][0]]
                    pairwise_contract[idx][1] = [pairwise_contract[idx][1]]
                pairwise_contract[idx][0].append(c[0])
                pairwise_contract[idx][1].append(c[1])
            else:
                pairwise_contract.append(list(c))
                tensor_pairs.append(tuple(np.sort((index_lookup[c[0]],index_lookup[c[1]]))))
                tensors_involved.add(index_lookup[c[0]])
                tensors_involved.add(index_lookup[c[1]])

    #Contract all internal indices
    for c in internal_contract:
        tensor_list[index_lookup[c[0]]].trace(c[0], c[1])

    #Contract pairs of tensors
    connected_component = [i for i in range(len(tensor_list))]
    for c in pairwise_contract:

        if isinstance(c[0], list):
            #Case where multiple indices of two tensors contracted
            d=index_lookup[c[0][0]]
            e=index_lookup[c[1][0]]
        else:
            d=index_lookup[c[0]]
            e=index_lookup[c[1]]

        if d==e:
            tensor_list[d].trace(c[0],c[1])
        else:
            if d<e:
                tensor_list[d]=tn.contract(tensor_list[d], tensor_list[e],
                        c[0], c[1])
                connected_component[e]=d
            else:
                tensor_list[e]=tn.contract(tensor_list[e], tensor_list[d],
                        c[1], c[0])
                connected_component[d]=e
            #Tensor in index_lookup refer to the first tensor
            #in which the label appers in the list
            for lab in tensor_list[min(d,e)].labels:
                index_lookup[lab]=min(d,e)

    #Take the tensor product of all the disconnected components
    return tn.tensor_product(*[tensor_list[connected_component.index(x)]
        for x in set(connected_component)])


#!/usr/bin/env python

from nornir import InitNornir
from nornir.core.filter import F
from nornir.plugins.tasks.networking import netmiko_send_command
from nornir.plugins.functions.text import print_result

DEFAULT_GATEWAY = ("10.220.88.1")

def exercise_3():
    """Get ip arp and format results
         takeaways: multigroup filters, netmiko_send_command, filtering parsed results
    """
    ios_filt = F(groups__contains="ios")
    eos_filt = F(groups__contains="eos")
    nr = InitNornir(config_file="config.yaml")
    nr = nr.filter(ios_filt | eos_filt)
    cmd = "show ip arp"
    result = nr.run(
        task=netmiko_send_command,
        command_string=cmd
    )

    parsed_res = []

    # get just the DG line
    for host, dd in result.items():
        output = dd[0].result
        for line in output.splitlines():
            if DEFAULT_GATEWAY in line:
                parsed_res.append({host:line}) # this is better done with a tuple, see exercise4

    # print table-like result
    # this is better done with a tuple, see exercise4
    for res in parsed_res:
        for k,v in res.items():
            print(f"Host: {k} || Gateway: {v}")

def main():
    exercise_3()

if __name__=="__main__":
    main()


# -*- coding: utf-8 -*-

import tensorflow as tf
import tensorlayer as tl
import tensorflow.contrib.slim as slim
import numpy as np

def boundary_process(boxes, img_shape):

    xmin, ymin, xmax, ymax = tf.unstack(boxes, axis=1)
    img_h, img_w = img_shape[1], img_shape[2]

    xmin = tf.maximum(xmin, 0.0)
    xmin = tf.minimum(xmin, tf.cast(img_w, tf.float32))

    ymin = tf.maximum(ymin, 0.0)
    ymin = tf.minimum(ymin, tf.cast(img_h, tf.float32))  # avoid xmin > img_w, ymin > img_h

    xmax = tf.minimum(xmax, tf.cast(img_w, tf.float32))
    ymax = tf.minimum(ymax, tf.cast(img_h, tf.float32))

    return tf.stack([xmin, ymin, xmax, ymax], axis=1)


# Ваша задача
# Написать скрипт для расчета корреляции Пирсона между
# двумя случайными величинами (двумя массивами). Можете
# использовать любую парадигму, но рекомендую использовать
# функциональную, т.к. в этом примере она значительно
# упростит вам жизнь.

from typing import List
from math import sqrt

def correlation_pearson(array_1: List[float], array_2: List[float]) -> float:
    """ Расчет корреляции Пирсона между двумя массивами. """

    if len(array_1) != len(array_2):
        raise ValueError("Длина массивов должна быть одинаковой")

    n = len(array_1)

    average_1 = sum(array_1) / n
    average_2 = sum(array_2) / n

    covariance = sum((array_1[i] - average_1) *
                     (array_2[i] - average_2) for i in range(n))
    variance_array_1 = sum((x - average_1) ** 2 for x in array_1)
    variance_array_2 = sum((y - average_2) ** 2 for y in array_2)

    correlation = covariance / \
        (sqrt(variance_array_1) * sqrt(variance_array_2))

    return round(correlation)

arr_1 = [1, 2, 3, 4, 5, 6, 7, 8, 9]
arr_2 = [9, 8, 7, 6, 5, 4, 3, 2, 1]

correlation = correlation_pearson(arr_1, arr_2)
print(f"Расчёт корреляции Пирсона = {correlation}")


import requests
import random
import lxml.etree as le
import xlwt

proxies_pool=[
    {'http':'202.109.157.61:9000'},
    {'http':'222.74.73.202:42055'},
    {'http':'202.109.157.60:9000'},
    {'http':'27.42.168.46:55481'},
    {'http':'182.139.111.149:9000'}
]
proxies=random.choice(proxies_pool)

headers={
    'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Mobile Safari/537.36 Edg/108.0.1462.76'
}

#url='https://m.ke.com/wh/loupan/p_whesscblhhv/'

url='https://m.ke.com/wh/loupan/'
resps=requests.get(url=url,headers=headers,proxies=proxies)
print(resps.status_code)
resps.encoding=resps.apparent_encoding
cont=resps.text

tr=le.HTML(cont)
page_link=tr.xpath('//div[@class="shell-resblock-card"]/a/@href')
#print(page_link)

title_data=[]
advantage_data=[]
price_data=[]
size_s_data=[]
type_s_data=[]
structure_data=[]
address_data=[]

for x in range(len(page_link)):
    link=page_link[x]
    new_link='https://m.ke.com'+link
    #print(new_link)
    #print(len(page_link))

    resp=requests.get(url=new_link,headers=headers,proxies=proxies)
    #print(resp.status_code)
    resp.encoding=resp.apparent_encoding
    con=resp.text

    tr=le.HTML(con)
    title=tr.xpath('//div[@class="title-wrapper resblock-name-line"]/h1/text()')
    advantage=tr.xpath('//div[@class="tag-wrapper"]/span/text()')
    price=tr.xpath('//*[@class="price-value item"]/text()')
    size_s=tr.xpath('//div[@class="address-value item"]/text()')[0]
    type_s=tr.xpath('//div[@class="address-value item"]/text()')[1]
    structure=tr.xpath('//div[@class="address-value item"]/text()')[2]
    address=tr.xpath('//div[@class="address-open-date"]/a/text()')

    title_data.append(title)
    advantage_data.append(advantage)
    price_data.append(price)
    size_s_data.append(size_s)
    type_s_data.append(type_s)
    structure_data.append(structure)
    address_data.append(address)

#print(advantage_data)

book=xlwt.Workbook(encoding='utf-8',style_compression=0)
sheet=book.add_sheet('武汉新房信息表',cell_overwrite_ok=True)
col=['名称','优势','均价','面积','户型','结构','地址']

#写入每一列
for i in range(7):
    sheet.write(0,i,col[i])

#写入每一行
for n in range(len(title_data)):
    sheet.write(n+1,0,title_data[n])
    sheet.write(n+1,1,advantage_data[n])
    sheet.write(n+1,2,price_data[n])
    sheet.write(n+1,3,size_s_data[n])
    sheet.write(n+1,4,type_s_data[n])
    sheet.write(n+1,5,structure_data[n])
    sheet.write(n+1,6,address_data[n])

book.save('E:\素材\素材\武汉新房信息表.xls')

print('save end')

自己独立完成的项目/爬取贝壳网.py


import sys
sys.stdin = open('input.txt')

# 나와야 하는 결과
#1 1  #2 4  #3 11
T = int(input()) # 3

for tc in range(1, T+1):

    # V: 노드수  E : 간선수
    text = input()
    # print(text)

    stack = []

    for word in text:
        stack.append(word)
        if len(stack) == 1:
            continue
        # 마지막 1,2번째 글자가 같으면 2개 pop
        elif stack[-1] == stack[-2]:
            # stack[len(stack)-1], stack[len(stack)-2]
            stack.pop()
            stack.pop()

    print(f'#{tc} {len(stack)}')


# 一万个小时学习Python
# 累计学习1个小时
# 开发时间：2022/5/5 14:33
a=4
for item in 'python':#第一次取出来是的P，把P赋值给item,第二次取出的是y，赋值给item
        a+=4
print(item)

#range()产生一个整数序列，————》也是一个可迭代数列

for i in range(1,10):
    print(i)

#如果用不到自定义变量，可将自定义变量缩写为“_”
for _ in range(5):
    print("人生苦短，我要学python")#0,1,2,3,4


from typing import Optional
from torch import nn, Tensor
import torch.nn.functional as F
import torch

__all__ = ["FocalCosineLoss"]

class FocalCosineLoss(nn.Module):
    """
    Implementation Focal cosine loss from the "Data-Efficient Deep Learning Method for Image Classification
    Using Data Augmentation, Focal Cosine Loss, and Ensemble" (https://arxiv.org/abs/2007.07805).

    Credit: https://www.kaggle.com/c/cassava-leaf-disease-classification/discussion/203271
    """

    def __init__(self, alpha: float = 1, gamma: float = 2, xent: float = 0.1, reduction="mean"):
        super(FocalCosineLoss, self).__init__()
        self.alpha = alpha
        self.gamma = gamma
        self.xent = xent
        self.reduction = reduction

    def forward(self, input: Tensor, target: Tensor) -> Tensor:
        cosine_loss = F.cosine_embedding_loss(
            input,
            torch.nn.functional.one_hot(target, num_classes=input.size(-1)),
            torch.tensor([1], device=target.device),
            reduction=self.reduction,
        )

        cent_loss = F.cross_entropy(F.normalize(input), target, reduction="none")
        pt = torch.exp(-cent_loss)
        focal_loss = self.alpha * (1 - pt) ** self.gamma * cent_loss

        if self.reduction == "mean":
            focal_loss = torch.mean(focal_loss)

        return cosine_loss + self.xent * focal_loss


from jinja2 import Environment, FileSystemLoader, select_autoescape
import yaml
import os
import sys
import argparse

'''
TODO:
  1. No try-catch blocks have been used to account for empty/uninitialized params in attributes.yaml

  2. Add basic unit/bdd tests (manually checked currently) & necessary code refactor to facilitate this:

     a. (positive) Verify creation of CSV for --generate_type=chart
     b. (positive) Verify creation of CSV using category name for --generate_type=experiment
     c. (positive) Verify creation of business logic, job & experiment CRs for --generate_type=experiment
     d. (negative) Verify validation/err-handle upon empty name, category attributes
'''

'''
NOTES:
  1. Category attribute is expected to match with chart names(though not mandatory), as per convention
     followed in litmuschaos/chaos-charts
'''

# generate_csv creates the chartserviceversion manifest
def generate_csv(csv_parent_path, csv_name, csv_config, litmus_env):
    csv_filename = csv_parent_path + '/' + csv_name + '.' + 'chartserviceversion.yaml'

    # Load Jinja2 template
    template = litmus_env.get_template('./templates/chartserviceversion.tmpl')
    output_from_parsed_template = template.render(csv_config)
    with open(csv_filename, "w+") as f:
        f.write(output_from_parsed_template)

# generate_chart creates the experiment-custom-resource manifest
def generate_chart(chart_parent_path, chart_config, litmus_env):
    chart_filename = chart_parent_path + '/' + 'experiment.yaml'

    # Load Jinja2 template
    template = litmus_env.get_template('./templates/experiment_custom_resource.tmpl')
    output_from_parsed_template = template.render(chart_config)
    with open(chart_filename, "w+") as f:
        f.write(output_from_parsed_template)

# generate_rbac creates the rbac for the experiment
def generate_rbac(chart_parent_path, chart_config, litmus_env):
    rbac_filename = chart_parent_path + '/' + 'rbac.yaml'

    # Load Jinja2 template
    template = litmus_env.get_template('./templates/experiment_rbac.tmpl')
    output_from_parsed_template = template.render(chart_config)
    with open(rbac_filename, "w+") as f:
        f.write(output_from_parsed_template)

# generate_engine creates the chaos engine for the experiment
def generate_engine(chart_parent_path, chart_config, litmus_env):
    engine_filename = chart_parent_path + '/' + 'engine.yaml'

    # Load Jinja2 template
    template = litmus_env.get_template('./templates/experiment_engine.tmpl')
    output_from_parsed_template = template.render(chart_config)
    with open(engine_filename, "w+") as f:
        f.write(output_from_parsed_template)

# generate_job creates the experiment job manifest
def generate_job(job_parent_path, job_name, job_config, litmus_env):
    job_filename = job_parent_path + '/' + job_name + '_' + 'k8s_job.yml'

    # Load Jinja2 template
    template = litmus_env.get_template('./templates/experiment_k8s_job.tmpl')
    output_from_parsed_template = template.render(job_config)
    with open(job_filename, "w+") as f:
        f.write(output_from_parsed_template)

# generate_ansible_logic creates the ansible_logic manifest
def generate_ansible_logic(ansible_logic_parent_path, ansible_logic_name, ansible_logic_config, litmus_env):
    ansible_logic_filename = ansible_logic_parent_path + '/' + ansible_logic_name + '_' + 'ansible_logic.yml'

    # Load Jinja2 template
    template = litmus_env.get_template('./templates/experiment_ansible_logic.tmpl')
    output_from_parsed_template = template.render(ansible_logic_config)
    with open(ansible_logic_filename, "w+") as f:
        f.write(output_from_parsed_template)

# generate_chaos_prerequisites creates the chaos_prerequisites manifest
def generate_chaos_prerequisites(chaos_prerequisites_parent_path, chaos_prerequisites_name, chaos_prerequisites_config, litmus_env):
    chaos_prerequisites_filename = chaos_prerequisites_parent_path + '/' + chaos_prerequisites_name + '_' + 'ansible_prerequisites.yml'

    # Load Jinja2 template
    template = litmus_env.get_template('./templates/experiment_ansible_prerequisites.tmpl')
    output_from_parsed_template = template.render(chaos_prerequisites_config)
    with open(chaos_prerequisites_filename, "w+") as f:
        f.write(output_from_parsed_template)

# generate_package creates the package manifest
def generate_package(package_parent_path, package_name):
    package_filename = package_parent_path + '/' + package_name + '.' + 'package.yaml'
    print(package_filename)
    with open(package_filename, "w+") as f:
        f.write('packageName: ' + package_name + '\n' + 'experiments:')

def main():
    # Required Arguments
    parser = argparse.ArgumentParser()
    parser.add_argument("-a", "--attributes_file", required=True,
                        help="metadata to generate chartserviceversion yaml")
    parser.add_argument("-t", "--generate_type", required=True,
                        help="scaffold a new chart or experiment into existing chart")
    # Optional Arguments
    parser.add_argument("-c", "--chart_name", required=False,
                        help="existing chart name to which experiment belongs, defaults to 'category' in attributes file")

    args = parser.parse_args()

    entity_metadata_source = args.attributes_file
    entity_type = args.generate_type
    entity_parent = args.chart_name

    # Load data from YAML file into a dictionary
    config = yaml.load(open(entity_metadata_source))
    entity_name = config['name']

    # Store the litmus root from bootstrap folder
    litmus_root = os.path.abspath(os.path.join("..", os.pardir))
    #env = Environment(loader = FileSystemLoader('./'), trim_blocks=True, lstrip_blocks=True, autoescape=True)
    env = Environment(loader = FileSystemLoader('./'), trim_blocks=True, lstrip_blocks=True, autoescape=select_autoescape(['yaml']))

    # if generate_type is chart, only create the chart(top)-level CSV & package manifests
    if entity_type == 'chart':
        chart_dir = litmus_root + '/experiments/' + entity_name
        if os.path.isdir(chart_dir) != True:
            os.makedirs(chart_dir)
        generate_csv(chart_dir, entity_name, config, env)
        generate_package(chart_dir, entity_name)

    # if generate_type is experiment, create the litmusbook arefacts (job, playbook, cr)
    elif entity_type == 'experiment':
        # if chart_name is not explicitly provided, use "category" from attributes.yaml as chart
        if entity_parent is None:
            experiment_category = config['category']
            chart_dir = litmus_root + '/experiments/' + experiment_category
        else:
            chart_dir = litmus_root + '/experiments/' + entity_parent
        # if a folder with specified/derived chart name is not present, create it
        if os.path.isdir(chart_dir) != True:
            os.makedirs(chart_dir)
            # generate csv for the freshly created chart folder
            generate_csv(chart_dir, experiment_category, config, env)

            # generate package for the freshly created chart folder
            generate_package(chart_dir, experiment_category)

        # create experiment folder inside the chart folder
        experiment_dir = chart_dir + '/' + entity_name
        if os.path.isdir(experiment_dir) != True:
            os.makedirs(experiment_dir)

        # generate experiment csv
        generate_csv(experiment_dir, entity_name, config, env)

        # generate experiment-custom-resource
        generate_chart(experiment_dir, config, env)

        # generate experiment specific rbac
        generate_rbac(experiment_dir, config, env)

        # generate experiment specific chaos engine
        generate_engine(experiment_dir, config, env)

        # generate experiment job
        generate_job(experiment_dir, entity_name, config, env)

        # generate chaos-ansible-logic
        generate_ansible_logic(experiment_dir, entity_name, config, env)

        # generate chaos-prerequisites
        generate_chaos_prerequisites(experiment_dir, entity_name, config, env)

if __name__=="__main__":
    main()


"""
Problem Statement #
Given ‘M’ sorted arrays, find the K’th smallest number among all the arrays.

Example 1:

Input: L1=[2, 6, 8], L2=[3, 6, 7], L3=[1, 3, 4], K=5
Output: 4
Explanation: The 5th smallest number among all the arrays is 4, this can be verified from the merged
list of all the arrays: [1, 2, 3, 3, 4, 6, 6, 7, 8]
Example 2:

Input: L1=[5, 8, 9], L2=[1, 7], K=3
Output: 7
Explanation: The 3rd smallest number among all the arrays is 7.
"""

from heapq import *

def find_Kth_smallest(lists, k):
    min_heap = []
    count = 1

    for i in range(len(lists)):
        heappush(min_heap, (lists[i][0], 0, lists[i]))

    while len(min_heap) > 0:
        val, curr_idx, l = heappop(min_heap)
        if count == k:
            return val
        count += 1
        next_idx = curr_idx + 1
        if next_idx < len(l):
            heappush(min_heap, (l[next_idx], next_idx, l))

    return -1


# Opus/UrbanSim urban simulation software.
# Copyright (C) 2010-2011 University of California, Berkeley, 2005-2009 University of Washington
# See opus_core/LICENSE

from numpy import where, logical_and
from urbansim.models.scaling_jobs_model import ScalingJobsModel

class DistributeUnplacedJobsModel(ScalingJobsModel):
    """This model is used to place randomly (within sectors) any unplaced jobs.
    """
    model_name = "Distribute Unplaced Jobs Model"

    def run(self, location_set, agent_set, agents_filter=None, **kwargs):
        """
            'location_set', 'agent_set' are of type Dataset. The model selects all unplaced jobs which pass the given agents_filter (if any),
            and passes them to ScalingJobsModel.
        """
        agents_index = agent_set.get_attribute(location_set.get_id_name()[0]) <= 0
        if agents_filter:
            dataset_pool = kwargs.get('dataset_pool', None)
            filter_values = agent_set.compute_variables([agents_filter], dataset_pool=dataset_pool)
            agents_index = logical_and(agents_index, filter_values>0)
        return ScalingJobsModel.run(self, location_set, agent_set, where(agents_index)[0], **kwargs)

from opus_core.tests import opus_unittest
from numpy import array, ma, arange, zeros
from urbansim.datasets.gridcell_dataset import GridcellDataset
from urbansim.datasets.job_dataset import JobDataset
from opus_core.storage_factory import StorageFactory

class Test(opus_unittest.OpusTestCase):
    def test_distribute_unplaced_jobs_model(self):
        # Places 1750 jobs of sector 15
        # gridcell       has              expected about
        # 1         4000 sector 15 jobs   5000 sector 15 jobs
        #           1000 sector 1 jobs    1000 sector 1 jobs
        # 2         2000 sector 15 jobs   2500 sector 15 jobs
        #           1000 sector 1 jobs    1000 sector 1 jobs
        # 3         1000 sector 15 jobs   1250 sector 15 jobs
        #           1000 sector 1 jobs    1000 sector 1 jobs
        # unplaced  1750 sector 15 jobs   0

        # create jobs

        storage = StorageFactory().get_storage('dict_storage')

        job_data = {
            "job_id": arange(11750)+1,
            "sector_id": array(7000*[15]+3000*[1]+1750*[15]),
            "grid_id":array(4000*[1]+2000*[2]+1000*[3]+1000*[1]+1000*[2]+1000*[3]+1750*[-1])
            }

        jobs_table_name = 'jobs'
        storage.write_table(table_name=jobs_table_name, table_data=job_data)

        jobs = JobDataset(in_storage=storage, in_table_name=jobs_table_name)

        storage = StorageFactory().get_storage('dict_storage')

        building_types_table_name = 'building_types'
        storage.write_table(
            table_name=building_types_table_name,
            table_data={
                "grid_id":arange(3)+1
                }
            )

        gridcells = GridcellDataset(in_storage=storage, in_table_name=building_types_table_name)

        # run model
        model = DistributeUnplacedJobsModel(debuglevel=4)
        model.run(gridcells, jobs)
        # get results

        # no jobs are unplaced
        result1 = where(jobs.get_attribute("grid_id")<0)[0]
        self.assertEqual(result1.size, 0)
        # the first 10000jobs kept their locations
        result2 = jobs.get_attribute_by_index("grid_id", arange(10000))
#            logger.log_status(result2)
        self.assertEqual(ma.allclose(result2, job_data["grid_id"][0:10000], rtol=0), True)

        # run model with filter
        # unplace first 500 jobs of sector 15
        jobs.modify_attribute(name='grid_id', data=zeros(500), index=arange(500))
        # unplace first 500 jobs of sector 1
        jobs.modify_attribute(name='grid_id', data=zeros(500), index=arange(7000, 7501))
        # place only unplaced jobs of sector 1
        model.run(gridcells, jobs, agents_filter='job.sector_id == 1')
        # 500 jobs of sector 15 should be unplaced
        result3 = where(jobs.get_attribute("grid_id")<=0)[0]
        self.assertEqual(result3.size, 500)
        # jobs of sector 1 are placed
        result4 = jobs.get_attribute_by_index("grid_id", arange(7000, 7501))
        self.assertEqual((result4 <= 0).sum(), 0)

if __name__=="__main__":
    opus_unittest.main()


import unittest

import requests
import responses

class TestCase(unittest.TestCase):

  @responses.activate
  def testExample(self):
    responses.add(**{
      'method'         : responses.GET,
      'url'            : 'http://example.com/api/123',
      'body'           : '{"error": "reason"}',
      'status'         : 404,
      'content_type'   : 'application/json',
      'adding_headers' : {'X-Foo': 'Bar'}
    })

    response = requests.get('http://example.com/api/123')

    self.assertEqual({'error': 'reason'}, response.json())
    self.assertEqual(404, response.status_code)


"""Given a string s containing just the characters '(', ')', '{', '}', '[' and ']', determine if the input string is valid.

An input string is valid if:

Open brackets must be closed by the same type of brackets.
Open brackets must be closed in the correct order.
Every close bracket has a corresponding open bracket of the same type."""

"""Example 1:

Input: s = "()"
Output: true

# started with open parentheses and closes in a closed parentheses.
# closed in the correct order.

Example 2:

Input: s = "()[]{}"
Output: true

# parentheses, brackets, and curly braces are open and closed in the
# correct order.

Example 3:

Input: s = "(]"
Output: false

# the order is off, they don't match each other.

Constraints:

1 <= s.length <= 104
s consists of parentheses only '()[]{}'."""

# O(n) = time complexity
# O(n) = space complexity

# solution
class Solution:
# create a function called isValid
    def isValid(self, s):
# initiate list
        parentheses_list = []
# create map for close to open parentheses
        close_to_open = { ")" : "(", "]" : "[", "}" : "{"}
# iterate through every character in the input string
        for char in s:
# if the character is in close_to_open map == closing parentheses
            if char in close_to_open:
# make sure our list is not empty
# cannot add a closing parentheses to an empty list
# check if the value at the top of our list is matching the opening parentheses
                if parentheses_list and parentheses_list[-1] == close_to_open[char]:
# if they match we can pop from our stack | removing an element at a specific position
                    parentheses_list.pop()
# if they don't match or the list is empty return false
                else:
                    return False
# if we get an open parentheses, take the char and append it to list
            else:
                parentheses_list.append(char)
# once we've gone through every char we can only return True
# if our list is empty otherwise we return false
        return True if not parentheses_list else False

if __name__ == "__main__":
    test = Solution()
    input = test.isValid("()[]{}")
    print(input)

# time complexity = O(n)
"""The time complexity of this algorithm is O(n), where n is the length of the input string.
This is because the algorithm iterates over each character in the string exactly once,
and the operations performed within the loop (appending to and popping from a list)
are constant time operations."""


'''
Author: Jerry9403 940341746@qq.com
Date: 2023-03-02 01:10:14
LastEditors: Jerry9403 940341746@qq.com
LastEditTime: 2023-03-02 01:10:38
FilePath: /leetcode/search-insert-position.py
Description: 这是默认设置,请设置`customMade`, 打开koroFileHeader查看配置 进行设置: https://github.com/OBKoro1/koro1FileHeader/wiki/%E9%85%8D%E7%BD%AE
'''
def searchInsert(self, nums, target):
        length = len(nums)
        left = 0
        right = length

        while(left < right - 1):
            cpm = (left + right) // 2
            if target == nums[cpm]:
                return cpm
            elif target > nums[cpm]:
                left = cpm
            else:
                right = cpm

        if target <= nums[left]:
            return left
        else:
            return right

        # 仍然是简单的二分查找


"""Pendulum Swing-up Environment with full observation."""
import numpy as np
from mbse.models.reward_model import RewardModel
from mbse.models.dynamics_model import DynamicsModel
from gym.envs.classic_control.pendulum import angle_normalize
import jax.numpy as jnp
import jax
from functools import partial
from typing import Union, Optional, Any
from mbse.utils.type_aliases import ModelProperties

from pyur5.models.ens_model import EnsembleModel
import math

class Ur5PendulumDynamicsModel(DynamicsModel):
    def __init__(self, task_typ='new', use_cos=True, ctrl_cost_weight=0.001, sparse=False, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.use_cos = use_cos
        self.task_typ = task_typ
        self.model = EnsembleModel(use_cos=use_cos, task_typ=task_typ)
        self.max_action = self.model.action_max
        self.min_action = self.model.action_min
        self.n_action = self.model.n_action
        self.n_obs = self.model.n_obs

        # TODO: Maybe think of punishing v_ee to be zero as well
        self.target_state = self.model.target_state #jnp.array([math.pi, 0]) # target state to be theta = pi/2, theta_dot = 0
        self.cost_weights = self.model.cost_weights[:-self.n_action] #jnp.array([1, 0.1]) # weight for theta and theta_dot

        # self.reward_model = Ur5PendulumReward(self.action_space, self.target_state, self.cost_weights, sparse=sparse)

        self.pred_diff = False
        self.obs_dim = self.model.x_dim #observation
        self.obs_space = self.model.obs_space

        print("model: ", self.model)
    def set_bounds(self, max_action, min_action=None):
        self.max_action = max_action
        if min_action is None:
            min_action = - max_action
        self.min_action = min_action
        self._init_fn()

    @partial(jax.jit, static_argnums=0)
    def predict(self,
                obs,
                action,
                rng=None,
                parameters=None,
                model_props: ModelProperties = ModelProperties(),
                sampling_idx: Optional[Union[jnp.ndarray, int]] = None,
                rescaled: bool = False,
                ):
        '''
            internal function to predict next state
        '''
        u = jnp.clip(action, self.min_action, self.max_action) # action is assumed to be in the range of [-1, 1]
        next_obs, reward, terminate, output_dict = self.model.step(obs, u)

        return next_obs

    def reset(self):
        '''
            return the initial observation
        '''
        if self.task_typ == 'new':
            if self.use_cos:
                obs = jnp.array([1.0, 0.0, 0.0] + [0.0, 0.0] * self.n_obs)
            else:
                obs = jnp.array([0.0, 0.0] + [0.0, 0.0] * self.n_obs)
        elif self.task_typ == 'sim':
            obs = jnp.array([1.0, 0.0, 0.0])

        return obs

    def evaluate(self,
                 obs,
                 action,
                 rng=None,
                 parameters=None,
                 sampling_idx=None,
                 model_props: ModelProperties = ModelProperties(),
                 rescaled: bool = False,
                 ):
        '''
            function to evaluate next state and reward
        '''
        # action here is assumed to be in the range of [-1, 1]
        # next_obs = self.predict(obs, action, rng, rescaled=False)
        # reward = self.model._reward_fn() #self.reward_model.predict(obs, action, next_obs)
        # return next_obs, reward
        next_obs, reward, _, _ = self.model.step(obs, action)
        return next_obs, reward

    # @staticmethod
    # @jax.jit
    # def _get_obs(state):
    #     theta, theta_dot, p_ee, v_ee = state[..., 0], state[..., 1], state[..., 2], state[..., 3]
    #     return jnp.asarray([theta, theta_dot, p_ee, v_ee], dtype=jnp.float32)

    # @partial(jax.jit, static_argnums=0)
    # def rescale_action(self, action):
    #     """Rescales the action affinely from  [:attr:`min_action`, :attr:`max_action`] to the action space of the base environment, :attr:`env`.

    #     Args:
    #         action: The action to rescale

    #     Returns:
    #         The rescaled action
    #     """
    # #     action = jnp.clip(action, self.min_action, self.max_action)
    #     low = self.model.action_space.low
    #     high = self.model.action_space.high
    #     action = low + (high - low) * (
    #             (action - self.min_action) / (self.max_action - self.min_action)
    #     )
    #     action = jnp.clip(action, low, high)
    #     return action

# class Ur5PendulumReward(RewardModel):
#     """Get Pendulum Reward."""

#     def __init__(self, action_space, target_sate, cost_weights, ctrl_cost_weight=0.001, sparse=False, *args, **kwargs):
#         super().__init__(*args, **kwargs)
#         self.ctrl_cost_weight = ctrl_cost_weight
#         self.sparse = sparse
#         self.min_action = min_action
#         self.max_action = max_action
#         self.action_space = action_space
#         self.target_state = target_state
#         self.cost_weights = cost_weights
#         self._init_fn()

#         print("target state: {}".format(self.target_state))
#         print("cost weights: {}".format(self.cost_weights))
#         print("reward action space: {}".format(self.action_space))

#     def _init_fn(self):
#         # rescale actions from [-1, 1] to [min_action, max_action]
#         self.rescale_action = jax.jit(lambda action: self._rescale_action(action=action,
#                                                                           min_action=self.min_action,
#                                                                           max_action=self.max_action,
#                                                                           low=self.action_space.low,
#                                                                           high=self.action_space.high,
#                                                                           ))
#         self.input_cost = jax.jit(lambda u: self._input_cost(u=u, ctrl_cost_weight=self.ctrl_cost_weight))

#         def predict(obs, action, next_obs=None, rng=None):
#             # return self.model._reward_fn(obs, action)
#             return self._predict(
#                 state_reward_fn=self.state_reward,
#                 input_cost_fn=self.input_cost,
#                 action_transform_fn=self.rescale_action,
#                 obs=obs,
#                 action=action,
#                 target_state=self.target_state,
#                 cost_weights=self.cost_weights,
#                 next_obs=next_obs,
#                 rng=rng
#             )

#         self.predict = jax.jit(predict)

#     def set_bounds(self, max_action, min_action=None):
#         self.max_action = max_action
#         if min_action is None:
#             min_action = - max_action
#         self.min_action = min_action
#         self._init_fn()

#     @staticmethod
#     @jax.jit
#     def state_non_sparse_reward(theta, theta_dot, p_ee, v_ee, target_state, cost_weights):
#         """Get sparse reward."""
#         theta = angle_normalize(theta)
#         dtheta = theta - target_state[0]
#         dtheta = angle_normalize(dtheta)
#         dtheta_dot = theta_dot - target_state[1]
#         cost = cost_weights[0] * (dtheta)**2 + cost_weights[1] * (dtheta_dot)**2
#         return -cost

#     @staticmethod
#     def _input_cost(u, ctrl_cost_weight):
#         # compute the |u|^2 * w
#         return ctrl_cost_weight * (jnp.sum(jnp.square(u), axis=-1))

#     @staticmethod
#     @jax.jit
#     def state_reward(state, target_state, cost_weights):
#         # TODO: This is outdated. NEED MODIFICATION!!
#         """Compute reward associated with state dynamics."""
#         # print("state shape: {}".format(state.shape))
#         # if state.shape[0] == 5:
#         if state.shape == (5,):
#             cos, sin, theta_dot, p_ee, v_ee = state[..., 0], state[..., 1], state[..., 2], state[..., 3], state[..., 4]
#             theta = jnp.arctan2(sin, cos)
#         else:
#             theta, theta_dot, p_ee, v_ee = state[..., 0], state[..., 1], state[..., 2], state[..., 3]

#         theta = angle_normalize(theta)
#         dtheta = theta - target_state[0]
#         dtheta = angle_normalize(dtheta) # normalize the angle to [-pi, pi]
#         dtheta_dot = theta_dot - target_state[1]
#         print("dtheta: {}".format(dtheta))
#         print("dtheta_dot: {}".format(dtheta_dot))
#         cost = cost_weights[0] * (dtheta)**2 + cost_weights[1] * (dtheta_dot)**2

#         # the reward is to make sure that we are
#         return -cost

#     @staticmethod
#     def _predict(state_reward_fn, input_cost_fn, action_transform_fn, obs, action, target_state, cost_weights, next_obs=None, rng=None, action_rng=[-1, 1]):
#         # action = action_transform_fn(action) # transform the action to the normal range
#         action = jnp.clip(action, action_rng[0], action_rng[1]) # clip to [-1, 1]
#         return state_reward_fn(state=obs, target_state=target_state, cost_weights=cost_weights) - input_cost_fn(action)

#     def evaluate(self,
#                  parameters,
#                  obs,
#                  action,
#                  rng,
#                  sampling_idx=None,
#                  model_props: ModelProperties = ModelProperties()):
#         print("Ur5DynamicsReward evaluate called")
#         next_state = self.predict(obs=obs, action=action, rng=rng)
#         print("next_state is : ", next_state)
#         reward = jnp.zeros(next_state.shape[0])
#         return next_state, reward

#     @staticmethod
#     def _rescale_action(action, min_action, max_action, low, high):
#         """
#         Args:
#             action: The action to rescale

#         Returns:
#             The rescaled action
#         """
#         # if min_action is not None and max_action is not None:
#         #     action = jnp.clip(action, min_action, max_action)
#         #     action = low + (high - low) * (
#         #             (action - min_action) / (max_action - min_action)
#         #     )
#         #     action = jnp.clip(action, low, high)
#         return action


"""Add member table

Revision ID: ee2d279cae47
Revises: e0e586eec464
Create Date: 2020-09-30 11:19:13.281166

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'ee2d279cae47'
down_revision = 'e0e586eec464'
branch_labels = None
depends_on = None

def upgrade():
    op.create_table(
        'member',
        sa.Column('id', sa.Integer(), primary_key=True, index=True),
        sa.Column('group_id', sa.Integer(), sa.ForeignKey("group.id", ondelete="CASCADE"), index=True, nullable=False),
        sa.Column('user_id', sa.Integer(), sa.ForeignKey("user.id", ondelete="CASCADE"), index=True, nullable=False),
        sa.Column('role', sa.Integer(), default=0, nullable=False),
        sa.Column('is_active', sa.Boolean(), default=False, nullable=False),
        sa.Column('created_at', sa.DateTime(), default=sa.sql.func.now(), nullable=False),
        sa.Column('updated_at', sa.DateTime(), default=sa.sql.func.now(), onupdate=sa.sql.func.now(), nullable=False),
    )

def downgrade():
    op.drop_table('member')


import numpy as np
import pytest

import pyswallow as ps
import pyswallow.handlers.boundary_handler as psbh

class TestMOSwallow:

    @pytest.fixture
    def swallow(self):
        bounds = {
            'x0': [-50.0, 50.0],
            'x1': [-50.0, 50.0]
        }

        swallow = ps.MOSwallow(bounds, n_obj=2)
        return swallow

    @pytest.fixture
    def opp_swallow(self):
        bounds = {
            'x0': [-50.0, 50.0],
            'x1': [-50.0, 50.0]
        }

        opp_swallow = ps.MOSwallow(bounds, n_obj=2)
        return opp_swallow

    def test_move(self, swallow):
        swallow.position = np.array([0.0, 0.0])
        swallow.velocity = np.array([10.0, 10.0])

        bh = psbh.StandardBH()
        swallow.move(bh)

        assert np.array_equal(swallow.position, swallow.velocity)

    def test_dominate(self, swallow, opp_swallow):
        opp_swallow.fitness = [50.0, 50.0]
        swallow.fitness = [5.0, 5.0]

        ret_bool = swallow.dominate(opp_swallow)

        assert ret_bool

    def test_self_dominate(self, swallow):
        swallow.fitness = [5.0, 5.0]
        swallow.pbest_fitness = [50.0, 50.0]

        ret_bool = swallow.self_dominate()

        assert ret_bool


from Graph import *
from AStar import *

A = LocationNode('A', 1, 4)
B = LocationNode('B', 3, 4)
C = LocationNode('C', 2, 2)
D = LocationNode('D', 4, 2)
S = LocationNode('S', 4, 1)
E1 = WeightedEdge(A, B, EuclideanDistance(A, B))
E2 = WeightedEdge(B, C, EuclideanDistance(B, C))
E3 = WeightedEdge(B, D, EuclideanDistance(B, D))
# E4 = WeightedEdge(B, S, euclideanDistance(B, S))
E5 = WeightedEdge(C, S, EuclideanDistance(C, S))
E6 = WeightedEdge(D, S, EuclideanDistance(D, S))

edgeList = [E1, E2, E3, E6, E5]
SetEdgeList(edgeList)
SetHeuristicDistanceFunc(EuclideanDistance)
solution = GetShortestPath(A, S)

for node in solution:
    print(node)


# logger.py - Set up a4h logging (sending trace output to stdout)
# 1. In main script...
#     start_log(logging.INFO)  # Call this once
# 2. In specific module...
#     import logging
#     ...
#     log = logging.getLogger('a4h')
#     ...
#     log.info('Listening on port:'+str(port))
#     ...
#     log.debug('SomeVar='+str(SomeVar))

import logging

def start_log(level, full_meta=False):
    """Create a unified a4h logger"""

    # Create a single logger to be used by all
    new_log = logging.getLogger('a4h')
    new_log.setLevel(level)

    # Create a console handler
    ch = logging.StreamHandler()
    ch.setLevel(level)
    ch.setFormatter(logging.Formatter('%(message)s [%(filename)s:%(lineno)d]'))
    new_log.addHandler(ch)

    if full_meta:
        # Add more context info to end of each log line
        ch.setFormatter(logging.Formatter(
            '"%(message)s"'
            + ',%(processName)s(%(process)d)'
            + ',%(threadName)s(%(thread)d)'
            + ',%(module)s:%(pathname)s:%(funcName)s:%(lineno)d'
            + ',%(asctime)s'
            + ',%(levelname)s')
        )


"""

@Time    : 6/15/21
@Author  : Wenbo
"""

import pandas as pd
import os

def read_jiahao_data(filename, is_only_vul=0):
    df = pd.read_csv(filename)
    df = df.fillna('')
    if is_only_vul == 1:
        df = df[ df['vul'] == 1]
    projects = {}
    for index, row in df.iterrows():
        proj = row['project'].lower().strip()
        cve_id = row['CVE ID'].strip()
        if proj == "" or cve_id == "":
            continue

        dd = {
            'func_name': row['func_name'].strip()+"()",
            'file_name': "./" + row['file_name'].strip(),
            'commit_id': row['commit_id'],
            'callee_num': int(row['callee_num']) if row['callee_num'] != '' else 0,
            'caller_num': int(row['caller_num']) if row['caller_num'] != '' else 0,
        }

        if proj not in projects.keys():
            projects[proj] = {}

        if cve_id not in projects[proj].keys():
            projects[proj][cve_id] = []

        projects[proj][cve_id].append(dd)

    return projects

def to_jiahao_data(jiahao_data, to_file):
    project_list = []
    cve_count_all_list = []
    cve_count_cc_list = []
    total_changed_functions_list = []
    total_changed_functions_cc_list = []
    total_callers_list = []
    total_callees_list = []
    for p in jiahao_data.keys():
        cves = jiahao_data[p]

        cve_count_all = len(cves.keys())
        cve_count_cc = 0

        total_changed_functions_all = 0
        total_changed_functions_cc = 0

        total_callers = 0
        total_callees = 0

        for cve_id in cves.keys():
            functions = cves[cve_id]

            total_changed_functions_all += len(functions)

            cve_has_cc = False
            for func in functions:
                total_callers += func['caller_num']
                total_callees += func['callee_num']
                if func['caller_num'] > 0 or func['callee_num'] > 0:
                    cve_has_cc = True
                    total_changed_functions_cc += 1
            if cve_has_cc:
                cve_count_cc += 1

        project_list.append(p)
        cve_count_all_list.append(cve_count_all)
        cve_count_cc_list.append(cve_count_cc)

        total_changed_functions_list.append(total_changed_functions_all)
        total_changed_functions_cc_list.append(total_changed_functions_cc)

        total_callers_list.append(total_callers)
        total_callees_list.append(total_callees)

    df = pd.DataFrame({
        'project': project_list,
        'cve_count(has commits)': cve_count_all_list,
        'cve_count(has callers or callees)': cve_count_cc_list,
        'total_functions(all changed)': total_changed_functions_list,
        'total_functions(has callers or callees)': total_changed_functions_cc_list,
        'total_callers': total_callers_list,
        'total_callees': total_callees_list
    })
    df.to_csv(to_file, index=False)
    print("saved to %s" % to_file)

def read_wenbo_data(filename):
    df = pd.read_csv(filename)
    df = df.fillna('')
    projects = {}
    for index, row in df.iterrows():
        proj = row['project'].lower().strip()
        cve_id = row['cve_id'].strip()
        if proj == "" or cve_id == "":
            continue

        dd = {
            'func_name': row['func_name'],
            'file_name': row['file_name'],
            'commit_id': row['commit_id'],
            'callee_num': int(row['callees_total_before']) if row['callees_total_before'] != '' else 0,
            'caller_num': int(row['callers_total_before']) if row['callers_total_before'] != '' else 0,
        }

        if proj not in projects.keys():
            projects[proj] = {}

        if cve_id not in projects[proj].keys():
            projects[proj][cve_id] = []

        projects[proj][cve_id].append(dd)

    return projects

def compare(jh_data, wb_data, to_file, jiahao_succ_cve_id_list):
    project_list = []
    cve_count_all_list = []
    cve_count_cc_list = []
    total_changed_functions_list = []
    total_changed_functions_cc_list = []
    total_callers_list = []
    total_callees_list = []
    total_commits_list = []

    is_new_project_list = []
    new_functions_list = []
    new_functions_cc_list = []
    new_cves_list = []
    new_cves_cc_list = []
    new_commits_list = []

    new_cves_unique = [] # 这里记一下相比 jiahao，我新找到的 cve 的。
    new_cves_cc_unique = [] # 这里记一下相比 jiahao，我新找到的有 cc 的 cve。

    for p in wb_data.keys():
        cves = wb_data[p]

        cve_count_all = len(cves.keys())
        cve_count_cc = 0

        total_changed_functions_all = 0
        total_changed_functions_cc = 0

        total_callers = 0
        total_callees = 0
        total_commits = 0

        new_funcs_count = 0    # new functions
        new_funcs_cc_count = 0 # new functions and has callers or callees

        new_cves_count = 0    # new cves
        new_cves_cc_count = 0 # new cves and has callers or callees

        new_commits_count = 0

        # 判断是否是新 project
        is_new_proj = 0
        if p not in jh_data.keys():
            is_new_proj = 1

        for cve in cves.keys():
            # 判断是否是新 cve
            is_new_cve = 0
            if is_new_proj:
                is_new_cve = 1
            else:
                if cve not in jh_data[p].keys():
                    is_new_cve = 1

            jh_func_names = []
            jh_commits = []
            if p in jh_data.keys() and cve in jh_data[p].keys():
                jh_funcs = jh_data[p][cve]
                for ff in jh_funcs:
                    if ff['func_name'] not in jh_func_names:
                        jh_func_names.append(ff['func_name'])
                    if ff['commit_id'] not in jh_commits:
                        jh_commits.append(ff['commit_id'])

            functions = cves[cve]

            total_changed_functions_all += len(functions)

            cve_has_cc = False
            commits_visted = []
            for func in functions:
                # 判读是否是 新 function
                is_new_func = 0
                if is_new_proj or is_new_cve:
                    is_new_func = 1
                if not is_new_cve and func['func_name'] not in jh_func_names:
                    is_new_func = 1

                total_callers += func['caller_num']
                total_callees += func['callee_num']
                if func['caller_num'] > 0 or func['callee_num'] > 0:
                    cve_has_cc = True
                    total_changed_functions_cc += 1
                    if is_new_func:
                        new_funcs_cc_count += 1

                # 判断是否是新 commit
                if not is_new_cve and func['commit_id'] not in commits_visted and func['commit_id'] not in jh_commits:
                    new_commits_count += 1
                    commits_visted.append(func['commit_id'])

                if is_new_cve and func['commit_id'] not in commits_visted:
                    new_commits_count += 1
                    commits_visted.append(func['commit_id'])

                if func['commit_id'] not in commits_visted:
                    commits_visted.append( func['commit_id'] )

                if is_new_func:
                    new_funcs_count += 1

            if cve_has_cc:
                cve_count_cc += 1
            if is_new_cve:
                new_cves_count += 1
                if cve not in new_cves_unique:
                    new_cves_unique.append(cve)
            if cve_has_cc and cve not in jiahao_succ_cve_id_list:
                new_cves_cc_count += 1
                if cve not in new_cves_cc_unique:
                    new_cves_cc_unique.append(cve)
            total_commits += len(commits_visted)

        project_list.append(p)
        cve_count_all_list.append(cve_count_all)
        cve_count_cc_list.append(cve_count_cc)

        total_changed_functions_list.append(total_changed_functions_all)
        total_changed_functions_cc_list.append(total_changed_functions_cc)

        total_callers_list.append(total_callers)
        total_callees_list.append(total_callees)
        total_commits_list.append(total_commits)

        is_new_project_list.append(is_new_proj)
        new_functions_list.append(new_funcs_count)
        new_functions_cc_list.append(new_funcs_cc_count)

        new_cves_list.append(new_cves_count)
        new_cves_cc_list.append(new_cves_cc_count)

        new_commits_list.append(new_commits_count)

    df = pd.DataFrame({
        'project': project_list,
        'cve_count(has commits)': cve_count_all_list,
        'cve_count(has callers or callees)': cve_count_cc_list,
        'total_functions(all changed)': total_changed_functions_list,
        'total_functions(has callers or callees)': total_changed_functions_cc_list,
        'total_callers': total_callers_list,
        'total_callees': total_callees_list,
        'total_commits': total_commits_list,

        'is_new_project': is_new_project_list,
        'new_functions': new_functions_list,
        'new_functions(has callers or callees)': new_functions_cc_list,
        'new_cves': new_cves_list,
        'new_cves(has callers or callees)': new_cves_cc_list,
        'new_commits': new_commits_list,
    })
    df.to_csv(to_file, index=False)
    print("saved to %s" % to_file)
    print("new_cves_cc_unique: %d" % len(new_cves_cc_unique))
    print("new_cves_unique: %d" % len(new_cves_unique))

if __name__ == '__main__':
    jiahao_file = "jiahao_data_with_func_name_no_code.csv"
    to_jiahao_file = "jiahao_data_project_level.csv"

    wenbo_file = "wenbo_data_with_commit_message2.csv"
    to_wenbo_file = "wenbo_data_project_level_v2.csv"

    # df = pd.read_csv(wenbo_file)
    # print(len(df['cve_id'].unique()))
    # exit()

    all_jiahao_cves = []
    df1 = pd.read_csv(jiahao_file)
    df1_succ = df1[(df1['callee_num'] > 0) | (df1['caller_num'] > 0)]
    jiahao_succ_cve_id_list = list(df1_succ['CVE ID'].unique())

    jiahao_data = read_jiahao_data(jiahao_file)
    if not os.path.exists(to_jiahao_file):
        to_jiahao_data(jiahao_data, to_jiahao_file)

    wenbo_data = read_wenbo_data(wenbo_file)
    compare(jiahao_data, wenbo_data, to_wenbo_file, jiahao_succ_cve_id_list)
    pass


import numpy as np
import matplotlib.pyplot as plt
import sys,os
import pickle
import csv

# PREAMBLE
def validate_input():
	if len(sys.argv) != 4:
    		print("Usage: python {} ANALYZED_DATA_FILE EPSILON LABEL".format(sys.argv[0]))
    		sys.exit(2)
	return sys.argv[1], float(sys.argv[2]), sys.argv[3]

def binning_analysis(O,k):
	# determine number of blocks
        noBlocks = len(O)//k
        N = k*noBlocks

        # Compute block averages
        O = np.array(O)
        Osplit = np.split(O[0:N],noBlocks)
        bas = np.mean(Osplit,axis=1)

        # Calculate block average and variance
        blockAvg = np.mean(bas)
        blockVar = np.var(bas)
        # Calculate block error
        blockError = np.sqrt(blockVar/noBlocks)

      	# Calculate average and variance of measurements
        Oavg = np.mean(O)
        Ovar = np.var(O)

      	# Calculate estimated autocorrelation time and effective statistics
        if Ovar > 0:
               	tINTo = k*blockVar/(2*Ovar)
        else:
                tINTo = float('nan')
        if tINTo > 0:
                Neff = N/(2*tINTo)
        else:
                Neff = float("nan")
        return Oavg, tINTo, Neff, blockError

def perform_binning_analysis(O):
    N = len(O)

    # Initialize variables for tracking data
    Neff = 0.0
    blockError = 0.0
    tINTo = 0.0

    for i in range(N//10):
        k = i+1
        Oavg, tINToNew, NeffNew, blockErrorNew = binning_analysis(O,k)
        if tINToNew > tINTo:
            Neff = NeffNew
            blockError = blockErrorNew
            tINTo = tINToNew
    data = [Oavg,tINTo,Neff,blockError]
    return data

# Load and organize data
datafile, epsilon, label = validate_input()
data = pickle.load(open(datafile,"r"))

def plot_ac(data,epsilon,label):
	Ts = [i[0] for i in data]

	a = np.array([i[5][0] for i in data])/8 # [angstrom]
	c = np.array([i[7][0] for i in data])/4 # [angstrom]
	a_err = np.array([i[5][3] for i in data])/8 # [angstrom]
	c_err = np.array([i[7][3] for i in data])/4 # [angstrom]

	m_a,b_a = np.polyfit(Ts,a,1)
	m_c,b_c = np.polyfit(Ts,c,1)
	x = np.linspace(Ts[0],Ts[-1],100)

	# Print
	print("a:")
	print("Slope: {}".format(m_a))
	print("Intercept: {}".format(b_a))
	print("c:")
	print("Slope: {}".format(m_c))
	print("Intercept: {}".format(b_c))

	plt.errorbar(Ts,a,yerr=a_err,fmt='-o',label="a")
	plt.errorbar(Ts,c,yerr=c_err,fmt='-o',label="c")
	plt.plot(x,m_a*x+b_a,label="a fit")
	plt.plot(x,m_c*x+b_c,label="c fit")
# Plot
plt.figure()
plt.title("Lattice Constants")
plot_ac(data,epsilon,label)
plt.legend()
plt.show()


import hypergraph as hg

graph1 = hg.Graph()
with graph1.as_default():
    d = hg.input_all().as_dict()
    hg.output() << [d.keys(), d.values(), d.items()]

ctx = hg.ExecutionContext()
with ctx.as_default():
    print(graph1({'a': 1, 'b': 2}))


import pytest
import re

# здесь должен быть импорт Решения студента
import author as user_module
# импорт строк, которые предоставляет платформа (для проверки работы тестов я создала temp.py
# с такими строками)
from temp import user_code, output

# Тестирование модуля
class TestUserModule():
    def test_class_exists(self):
        try:
            from author import Contact
        except ImportError:
            assert False, 'Убедитесь, что класс Contact cуществует (проверьте нет ли опечаток)'
    def test_func_not_exists(self):
        try:
            from author import print_contact
        except ImportError:
            assert True, 'Убедитесь, что удалили функцию print_contact'

    def test_methods_exist(self):
        assert '__init__' in user_module.Contact.__dict__ , 'Проверьте, что вы не удалили метод __init__ в классе Contact'
        assert 'show_contact' in user_module.Contact.__dict__ , (
            'Убедитесь, что метод show_contact cуществует (проверьте нет ли опечаток)'
        )

    def test_objcets(self):
        try:
            from author import mike, vlad
        except ImportError:
            assert False, 'Убедитесь, что cоздали объекты mike и vlad'
        assert isinstance(user_module.mike, user_module.Contact), (
            'Убедитeсь что объект mike является экземпляром класса Сontact')
        assert isinstance(user_module.vlad, user_module.Contact), (
            'Убедитeсь что объект vlad является экземпляром класса Сontact'
        )
        parametres = ['name', 'phone', 'birthday', 'address']
        for param in parametres:
            assert hasattr(user_module.vlad, param) and hasattr(user_module.mike, param), (
                f'Проверьте, что вы не удалили необходимые параметры {parametres}'
                f'в классе и при создании объктов mike и vlad передали их'
            )

    def test_print_funcs_work(self, capsys):
        vlad = user_module.Contact("Владимир Маяковский", "73-88", "19.07.1893", "Россия, Москва, Лубянский проезд, д. 3, кв. 12")
        user_module.vlad.show_contact()
        capture = capsys.readouterr()
        correct_output = [
        'Создаём новый контакт Владимир Маяковский',
        'Владимир Маяковский — адрес: Россия, Москва, Лубянский проезд, д. 3, кв. 12, телефон: 73-88, день рождения: 19.07.1893'
    ]
        problem1 = ('Убедитесь, что при создании экземпляра класса выводите'
                   '`Создаём новый контакт <name>`')
        problem2 = ('Убедитесь, что при вызове метода show_contact выводите'
                    'все параметры из прекода в нужном порядке')
        assert correct_output[0] in capture.out, problem1
        assert correct_output[1] in capture.out, problem2

# Тестирование строк user_code и output
# (если я правильно поняла условие, эти строки передаются платформой)
def test_student_code():
    patterns = {
    r'class Contact' : 'Проверьте, что создали класс Contact',
    r'def __init__' : 'Проверьте, что вы не удалили метод __init__ в классе Contact',
    r'print\(f"Создаём новый контакт {name}"\)': 'Проверьте, что вы не удалили print из метода __init__ в классе Contact',
    r'def show_contact\(self\)' : 'Убедитесь, что создали мeтод show_contact с параметром self',
    r'print\(f"{self\.name} — адрес: {self\.address}, телефон: {self\.phone}, день рождения: {self\.birthday}"\)':
     'Убедитесь, что в методе show_contact вы выводите строку из функции print_contact, заменив имя конкретного объекта на `self`',
    r'mike\s*=\s*Contact\("Михаил Булгаков",\s*"2-03-27",\s*"15\.05\.1891",\s*"Россия, Москва, Большая Пироговская, дом 35б, кв\. 6"\)':
     'Убедитесь, что создали экземпляр класса Contact mike с параметрами из прекода',
    r'vlad\s*=\s*Contact\("Владимир Маяковский",\s*"73-88",\s*"19\.07\.1893",\s*"Россия, Москва, Лубянский проезд, д\. 3, кв\. 12"\)':
     'Убедитесь, что создали экземпляр класса Contact vlad с параметрами из прекода',
    r'vlad\.show_contact\(\)':'Убедитесь, что вызвали метод show_contact для объекта vlad',
    r'mike\.show_contact\(\)':'Убедитесь, что вызвали метод show_contact для объекта mike'
}
    for key in patterns:
        assert re.findall(f'{key}', user_code) != [], patterns[key]

    assert re.findall(r'def print_contact', user_code) == [], 'Убедитесь, что удалили функцию print_contact'

def test_student_output():
    correct_output = [
        'Создаём новый контакт Михаил Булгаков',
        'Создаём новый контакт Владимир Маяковский',
        'Михаил Булгаков — адрес: Россия, Москва, Большая Пироговская, дом 35б, кв. 6, телефон: 2-03-27, день рождения: 15.05.1891',
        'Владимир Маяковский — адрес: Россия, Москва, Лубянский проезд, д. 3, кв. 12, телефон: 73-88, день рождения: 19.07.1893'
    ]
    problem1 = ('Убедитесь, что при создании экземпляра класса выводите'
                '`Создаём новый контакт <name>`')
    problem2 = ('Убедитесь, что при вызове метода show_contact выводите'
                'все параметры из прекода в нужном порядке')
    assert correct_output[0] in output, problem1
    assert correct_output[1] in output, problem1
    assert correct_output[2] in output, problem2
    assert correct_output[3] in output, problem2


from __future__ import annotations

import pytest

from subtitles_translator.available_languages import AvailableLanguages
from subtitles_translator.translator import Subtitles, Translator

@pytest.fixture
def raw_subtitles() -> list[str]:
    raw_str = """
1
00:00:00,498 --> 00:00:02,827
Hello World!
"""
    raw_subtitles = raw_str.split("\n")

    return raw_subtitles

@pytest.fixture
def subtitles(raw_subtitles) -> Subtitles:
    subtitles = Subtitles(raw_subtitles)

    return subtitles

@pytest.fixture
def translator() -> Translator:
    translator = Translator(AvailableLanguages("en"), AvailableLanguages("fr"))

    return translator

def test_translate():
    translator = Translator(source_language=AvailableLanguages("en"), target_language=AvailableLanguages("fr"))
    source_test = "Hello World!"
    translated_test = translator.translate(source_test)
    assert translated_test == "Bonjour le monde!"

def test_translate_subtitles(translator, subtitles):
    translator.translate_subtitles(subtitles)
    assert subtitles.full_text_lines[3] == "Bonjour le monde!"


from random import choice

from emoji import emojize
from telegram import ReplyKeyboardMarkup, KeyboardButton
from utils import *

import settings

def get_user_emoji(user_data):
    if 'emoji' in user_data:
        return user_data['emoji']
    else:
        user_data['emoji'] = emojize(choice(settings.USER_EMOJI), use_aliases=True)
        return user_data['emoji']

def get_keyboard():
    contact_button = KeyboardButton('Отправить контакты', request_contact=True)
    location_button = KeyboardButton('Отправить геолокацию', request_location=True)
    my_keyboard = ReplyKeyboardMarkup([
                                        ['Хатю котика', 'Сменить аватар'],
                                        [contact_button, location_button]
                                       ], resize_keyboard=True
                                      )
    return my_keyboard


# Given an array of size n, find the majority element. The majority element is the element that appears more than ⌊ n/2 ⌋ times.
#
# You may assume that the array is non-empty and the majority element always exist in the array.

#用hash
class Solution(object):
    def majorityElement(self, nums):
        """
        :type nums: List[int]
        :rtype: int
        """
        le = len(nums)
        hash = {}
        for num in nums:
            if hash.has_key(num):
                hash[num] += 1
                if hash[num] > le / 2:
                    return num
            else:
                hash[num] = 1
                if hash[num] > le / 2:
                    return num

class Solution(object):
    def majorityElement(self, A):
        # class Solution(object):
        # def majority(A):
        le = len(A)
        left = self.helper(A)
        if left != [] and A.count(left[0]) > le / 2:
            return left[0]
        else:
            return None

    def helper(self, A):
        le = len(A)
        if le == 2:
            if A[0] != A[1]:
                return []
            else:
                return A
        if le == 1:
            return A
        left = self.helper(A[:le / 2 + 1])
        right = self.helper(A[le / 2 + 1:])
        if len(left) == len(right):
            if left == [] or left[0] != right[0]:
                return []
            else:
                return left + right
        else:
            i = 0
            j = 0
            while i < len(left) and j < len(right):
                if left[i] != right[j]:
                    del left[i]
                    del right[j]
                else:
                    break
            return left + right

def majorityElement(nums):
    count = 0
    candidate1 = 0
    for num in nums:    #找到挨着的一样的数,这个数出现的次数还大于一半
        if candidate1 == num:
            count +=1
        elif count == 0:
            candidate1 = num
            count +=1
        elif candidate1 != num:
            count-=1
    fre = nums.count(candidate1)
    if fre>len(nums)/2:
        return candidate1
    else:
        return -1