import streamlit as st
import pandas as pd
import datetime
from PIL import Image
import matplotlib.pyplot as plt
st.title("Hello World")
st.text('Bu bir text')
st.markdown('# This is a markdown text')
st.markdown('`this is a markdown text`')
st.header('this is a header')
st.subheader('this is subheader')
st.success('Congrats')
st.info('Warning!!!')
st.error('Error')
st.help(map)
st.write('this is a text written with write function')
img = Image.open('cat.jpg')
st.image(img,caption = 'Cat',use_column_width = True)
st.video('https://www.youtube.com/watch?v=eX2qFMC8cFo')
#checkbox
if st.checkbox('hide and seek'):
    st.write('Seek')

status = st.radio('What is your favorite color?',('red','green','blue'))
st.write(f'your favorite color is {status}')
button = st.button('press me!!!')
if button:
    st.success('button just clicked')

number = st.selectbox('select a numver',(1,2,3,4,5))
st.write(f'you choose {number}')

st.multiselect('select multiple numbers',[1,2,3])

st.slider('Select a number',0,9,3,2) # 3 default, 2 stepsize

name = st.text_input('Enter Your Name')
st.write(f'your name is {name}')

if st.button('Submit'):
    st.write(f'Hello {name.swapcase()} ')

st.text_area('Enter a message','let go')

st.date_input('date')

st.time_input('time')

st.code('import pandas as pd')

with st.echo():
    import numpy as np
    import pandas as pd
    import matplotlib.pyplot as plt
    df = pd.DataFrame({'a': np.random.randint(1,10,5),
                      'b':np.arange(0,10,2)})
    df

st.sidebar.title('Side Bar')
st.sidebar.text_area('text in side bar')
st.sidebar.slider('input',0,5,1,1)

df = pd.read_csv('Mall_Customers.csv')
st.table(df.head(10))
st.sidebar.table(df.head().iloc[:3,:3])
st.write(df.head())  # dinamik

st.dataframe(df.head())

st.line_chart(df.Age)

slider = st.slider('x')
seri = np.random.randn(slider)
st.line_chart(seri)
st.write(seri.plot())


"""
* Assignment: Sequence Slice Substr
* Complexity: easy
* Lines of code: 3 lines
* Time: 5 min

English:
    1. Use data from "Given" section (see below)
    2. Use `str.find()` and slicing
    3. Print `TEXT` without text in `REMOVE`
    4. Compare result with "Tests" section (see below)

Polish:
    1. Użyj danych z sekcji "Given" (patrz poniżej)
    2. Użyj `str.find()` oraz wycinania
    3. Wypisz `TEXT` bez tekstu z `REMOVE`
    4. Porównaj wyniki z sekcją "Tests" (patrz poniżej)

Tests:
    >>> type(result)
    <class 'str'>
    >>> result
    'We choose the Moon!'
"""

# Given
TEXT = 'We choose to go to the Moon!'
REMOVE = 'to go to '

# Solution
a = TEXT.find(REMOVE)   # 10
b = a + len(REMOVE)     # 19
result = TEXT[:a] + TEXT[b:]


# -*- coding: utf-8 -*-
# __author__ = 'zoujiaxun'

import os
from biocluster.workflow import Workflow
import datetime
import unittest
import types
from bson.objectid import ObjectId

class VennWorkflow(Workflow):
    """
    plot venn

    """
    def __init__(self, wsheet_object):
        self._sheet = wsheet_object
        super(VennWorkflow, self).__init__(wsheet_object)
        options = [
            {"name": "venn_file", "type": "infile", "format": "ref_rna_v2.common"},# 用户上传文件
            {'name': 'sep', 'type': 'string'},  # 表格的分隔符
            {'name': 'update_info', 'type': 'string'},
            {'name': "main_id", 'type': 'string'},
        ]
        self.add_option(options)
        self.revise_infiles()
        self.set_options(self._sheet.options())

    def check_options(self):
        # for k, v in self.sheet.options().items():
        #     self.logger.debug('{} = {}'.format(k, v))
        pass

    def run(self):
        self.run_venn()
        super(VennWorkflow, self).run()

    def run_venn(self):
        self.venn = self.add_tool('tool_lab.venn')
        self.venn.set_options({
            'venn_file': self.option('venn_file').prop["path"],
            'sep':self.option('sep')
        })
        self.venn.on('end',self.set_db)
        self.venn.run()

    def end(self):
        super(VennWorkflow, self).end()

    def set_db(self):
        """
        保存结果表到mongo数据库中
        """
        venn = self.api.api("tool_lab.venn")
        # add result info
        graph_table = os.path.join(self.venn.output_dir, 'venn_graph.xls')
        venn.add_venn(graph_table, main_id=self.option('main_id'), )
        self.end()

class TestFunction(unittest.TestCase):
    '''
    This is test for the workflow. Just run this script to do test.
    '''

    def test(self):
        from mbio.workflows.tool_lab.venn import VennWorkflow
        from biocluster.wsheet import Sheet
        import random
        # test_dir = "/mnt/ilustre/users/sanger-dev/app/database/Genome_DB_finish/fungi/Saccharomyces_cerevisiae/Ensemble_release_39/"
        data = {
            "id": "venn" + str(random.randint(1, 10000)),
            "type": "workflow",
            "name": "tool_lab.venn",
            "options": dict(
                # genome=test_dir + "/" + "dna/Saccharomyces_cerevisiae.dna.toplevel.fa",
                # fasta_seq=">scaffold1\nATTATAAAATGAGGAGATGTAAATTTTAAGGAAAAAAATAAAGCTGTCGAGATTTTCTCGACAGCCTGAAGAGCACCCATCATAAAAGGGTGCTCTTTTTCTTTTCTCTTCCCGATTTGTTCACAGGCTGAATCCTCTCCTCATATGCTGAAAGGGAGATTCAGAAAATATTACAGACTTTTTTAAATAAAGAAGGTGAACGGTCAGTATGTTGAGCGGTTTAACGGTTGCGGTGATCGGGGGAGATGCAAGGCAGCTTGAAATCATTCGCAAGCTGTCACAGCAGCATGCCAAAGTGTTTTTGGTCGGATTTGATCAGCTGGATCATGGGTTTATCGGTGCTGAAAAGCTTAAAATGTCAGAACTTCCATTTGAACAGGTAGACAGTATGATTCTGCCGGTATCAGGTGCAACAGATGAAGGCGTCGTCGCCACAGTTTTCTCAAATGAGCAGGTCGTGCTGGAAGCAGAATATTTAGAAAGAACTCCAGCACATTGTACCTTGTACTCAGGTATTTCTAATACGTACTTAGACAATCTGGCAAAGCAGGTGAACCGGAAGCTTGTGAAGCTGTTTGAGCGCGATGATATTGCCATATATAACTCTATTCCAACAGTTGAAGGGATTATCATGATGGCCATTCAGCAAACGGACTATACGATTCATGGATCACATGTCGCTGTCCTCGGGCTTGGGAGAACAGGGCTCACAATTGCCCGCACAT",
                venn_file="/mnt/ilustre/users/sanger-dev/sg-users/zoujiaxun/transcript.count.matrix",
                # fasta_file="/mnt/ilustre/users/sanger-dev/sg-users/fuwenyao/tool_lab/seq_filter/input/example.fasta",
                # fasta_file="",
                # target_type="png",
                # max_len='10000'
            )
        }

        wsheet = Sheet(data=data)
        wf =VennWorkflow(wsheet)
        wf.sheet.id = 'venn'
        wf.sheet.project_sn = 'venn'
        wf.IMPORT_REPORT_DATA = False
        wf.IMPORT_REPORT_AFTER_DATA = False
        wf.run()

if __name__ == '__main__':
    suite = unittest.TestSuite()
    suite.addTests([TestFunction('test')])
    unittest.TextTestRunner(verbosity=2).run(suite)


for _ in range(int(input())):
    N=int(input())
    ans=0
    for i in range(9,1,-1):
        while N%i==0:
            # 나는 while문을 for문 바깥에 뒀다면 이 분은 안쪽에 두었음
            # 하나의 수를 가지고 N을 나누어지지 않을 때까지 계속 나누는 방식
            N=N//i
            ans+=1
            # 따로 리스트를 사용할 필요없이 ans 변수 하나만으로 충분 -> 어차피 개수만 필요하기 때문
    if ans==0: ans+=1
    # 1이 들어왔을 때 예외 처리
    print(ans if N==1 else -1)


def gcd(a,b):
    if a %b==0:
        return b
    else:
        return gcd(b,a%b)

def lcm(a,b):
    hcf=gcd(a,b)
    return (hcf*(a//hcf)*(b//hcf))

a,b=eval(input("DIVIDEND , DIVISOR ="))
print(f"GCD of {a} and {b} is : {gcd(a,b)}")
print(f"LCM of {a} and {b} is : {lcm(a,b)}")


import logging
from pprint import pformat, pprint
from flask_jsonrpc.proxy import ServiceProxy

logger = logging.getLogger(__name__)

AZURE = "http://ftr-app.japanwest.cloudapp.azure.com:5000"
LOCAL = "http://localhost:5000"

SERVER_URL = AZURE
API_URI = '/api'

server = ServiceProxy(SERVER_URL + API_URI)

def call_debug(func,*args,**kwargs):
	def wrap():
		print("")
		print("-"*80)
		print(" METHOD - {}".format(func.__name__))
		print(" API_SERVER - {}".format(SERVER_URL))
		print("-"*80)
		r = func(*args,**kwargs)
		print("[ result ]")
		print(pformat(r))
		print("")
	return wrap

def rpc_call(func,*args,**kwargs):
    try:
        logger.info("--CALL")
        r = func(*args,**kwargs)
        logger.info(r)
        if 'error' in r:
            return r.get('error').get('name')
        else:
            return r.get('result')
    except Exception as e:
        logger.error(e)
        return None


import os
import pdb
import csv
import numpy as np
import cv2
import torch
import pandas as pd
from tqdm import tqdm
import scipy.io as sio
from spconv.pytorch.utils import PointToVoxel
data_path = "xxx"
save_path = "xxx"
video_files = os.listdir(data_path)

def transform_points_to_voxels(data_dict = {}, device = torch.device("cuda:0")):
    points = data_dict['points']
    # 将points打乱
    shuffle_idx = np.random.permutation(points.shape[0])
    points = points[shuffle_idx]
    data_dict['points'] = points

    voxel_generator = PointToVoxel(
        # 给定每个voxel的长宽高
        vsize_xyz=[10, 10, 10],
        # 给定的范围
        coors_range_xyz=[0, 0, 0, 200, 239, 179],
        # 给定每个event的特征维度 t,x,y,p
        num_point_features=4,
        # 最多选取多少个voxel，
        max_num_voxels=10000,
        # 给定每个voxel中有采样多少个点，不够则补0
        max_num_points_per_voxel=8,  # 32
        device=device
    )
    # 使用spconv生成voxel输出
    points = torch.tensor(data_dict['points'])
    points = points.to(device)
    voxel_output = voxel_generator(points)

    voxels, coordinates, num_points = voxel_output
    # pdb.set_trace()
    voxels = voxels.to(device)
    coordinates = coordinates.to(device)
    num_points = num_points.to(device)
    # 选event数量在前5000的voxel
    if num_points.shape[0] < 512:
        features = voxels[:,:,3]
        coor = coordinates[:,:]
    else:
        _, voxels_idx = torch.topk(num_points, 512)
    # 将每个voxel的1024个p拼接作为voxel初始特征
        features = voxels[voxels_idx][:, :, 3]
    # 前5000个voxel的三维坐标
        coor = coordinates[voxels_idx]
    # 将y.x.t改为t,x,y
    coor[:, [0, 1, 2]] = coor[:, [2, 1, 0]]

    return coor, features

if __name__ == '__main__':
    device = torch.device("cuda:0")
    video_files = os.listdir(data_path)

    label_dict = {lbl: i for i, lbl in enumerate(video_files)}
    for videoID in range(len(video_files)):
        videoName = video_files[videoID]
        fileLIST = os.listdir(os.path.join(data_path, videoName))
        if not os.path.exists(os.path.join(save_path,  videoName)):
            os.makedirs(os.path.join(save_path,  videoName))
        mat_save = os.path.join(save_path,  videoName)
        for FileID in tqdm(range(len(fileLIST))):
            file = fileLIST[FileID]
            save_name = file.split('.')[0]
            if os.path.exists(os.path.join(mat_save,'{}.mat'.format(save_name))):
                continue
            label = label_dict.get(videoName, None)
            read_path = os.path.join(data_path, videoName, file)
            mat_file = sio.loadmat(read_path)
            t = mat_file["ts"]
            time_length = t[-1]-t[0]
            t = ((t-t[0]) / time_length) * 200.0
            x = mat_file["x"].astype(np.float64)
            y = mat_file["y"].astype(np.float64)
            p = mat_file["pol"].astype(np.float64)
            p[np.where(p==0)]=-1
            all_events = np.hstack((t,x,y,p))
            all_events = torch.from_numpy(all_events).float()
            all_events = all_events.to(device)
            data_dict = {'points': all_events}
            coor, features = transform_points_to_voxels(data_dict=data_dict, device=device)
            coor = coor.cpu()
            features = features.cpu()
            coor = coor.numpy()
            features = features.numpy()
            sio.savemat(os.path.join(mat_save , '{}.mat'.format(save_name)), mdict={'coor': coor, 'features': features})


from captcha.image import ImageCaptcha
import random

import captcha

if __name__ == "__main__":
    # Create an image instance of the given size
    sizes = []
    for i in range(1, 10):
        sizes.append(random.randint(110, 170))

    imageCaptcha = ImageCaptcha(fonts=["./storage/fonts/captchaFont.ttf"], font_sizes=sizes, width=500, height=300)
    # ImageCaptcha.create_noise_dots(image, color = (0, 0, 0), width = 1, number = 100)

    possibleLetters = "ABCEFGHJKLMNOPRSTUVWXYZ"

    captchaStr = " "
    for i in range(0, 5):
        captchaStr += random.choice(possibleLetters)
        captchaStr += " " * random.randint(1, 4)

    # generate the image of the given text
    data = imageCaptcha.generate(captchaStr)
    # image.create_noise_dots(image, color = (0, 0, 0), width = 1, number = 100)
    print(captchaStr)

    # write the image on the given file and save it
    imageCaptcha.write(captchaStr, "CAPTCHA.png")


import sys
import json
import requests
import re

# Function for HTTPRequestAction call
def requestAction(action, data):
    url = action['options']['url']
    # if url contains interpolated values, use function
    if "{{" and "}}" in url:
        url = interpolation(url, storedData)
    r = requests.get(url)
    # if 2xx HTTP response gather request data
    if r.status_code == 200 or 204:
        storedData2 = r.text
        parse_json = json.loads(storedData2)
        return parse_json
    # implement Network failure and termination for non-2xx HTTP response
    else:
        print('Network Failure. Terminating Program')
        sys.exit()

# Function to interpolate values to string/url
def interpolation(string, inputData):
    # identify and store values between dounble curly braces
    text_in_brackets = re.findall('[^{\{]+(?=}\})', string)
    keys = []
    values = []
    # loop to split values
    for i in text_in_brackets:
        keys.append(i.split("."))
    # double loop to identify and store keys values from HTTPRequest data
    for j in range(0, len(keys)):
        temp = storedData
        for k in range(0, len(keys[j])):
            # try except to store found data and store blank if key not valued
            try:
                temp = temp[keys[j][k]]
            except:
                temp = ''
        values.append(temp)
    # iterate through string and replace curly braces keys with stored values
    for l in range(0, len(text_in_brackets)):
        string = string.replace(
            "{{"+str(text_in_brackets[l])+"}}", str(values[l]))
    return string

# locate json file from command line argument
filename = "datafiles/" + sys.argv[1]
try:
    with open(filename) as json_file:
        # gather data from data file and convert to json format
        actionsObject = json.load(json_file)
        # object to store all HTTPRequestAction data
        storedData = {}
        # iterate through actions to store & trigger function depending on type
        for action in actionsObject['actions']:
            if action['type'] == "HTTPRequestAction":
                # Request data and store with actions name as value
                storedData[action['name']] = requestAction(action, storedData)
            if action['type'] == "PrintAction":
                # Gather string, interpolate and print to console
                printString = action['options']['message']
                message = interpolation(printString, storedData)
                print(message)
# implement Network failure and termination if file not located
except FileNotFoundError:
    print('Invalid file name.')
    sys.exit()


# -*- coding: utf-8 -*-
import scrapy
import re
from MoguSpider.items import SuningSpiderItem
import datetime

class SuningSpider(scrapy.Spider):
    name = 'suning'
    allowed_domains = ['list.suning.com']
    start_urls = ['https://list.suning.com/']

    def parse(self, response):
        search_divs = response.css(".search-main > div")
        for search_div in search_divs:
            firstId = search_div.css("div::attr(id)").extract_first("")
            firstName = search_div.css("h2::text").extract_first("")
            suningItem = SuningSpiderItem()
            suningItem["Name"] = firstName
            suningItem["SourceId"] = firstId
            suningItem["ParentId"] = "0"
            suningItem['CreatedDate'] = datetime.datetime.now()
            suningItem['SyncTime'] = datetime.datetime.now()
            yield suningItem
            boxs = search_div.css(".title-box")
            for box in boxs:
                secondUrl = box.css(".t-left a::attr(href)").extract_first("")
                martch_re = re.match(".*-(\d+)-.*",secondUrl);
                secondId = martch_re.group(1)
                secondName = box.css(".t-left a::attr(title)").extract_first("")
                rights = box.css(".t-right a")
                suningItem["Name"] = secondName
                suningItem["SourceId"] = secondId
                suningItem["ParentId"] = firstId
                suningItem['CreatedDate'] = datetime.datetime.now()
                suningItem['SyncTime'] = datetime.datetime.now()
                yield suningItem
                for right in rights:
                    thirdUrl = right.css("a::attr(href)").extract_first("")
                    martch_re = re.match(".*-(\d+)-.*",thirdUrl);
                    thirdId = martch_re.group(1)
                    thirdName = right.css("a::text").extract_first("")
                    suningItem["Name"] = thirdName
                    suningItem["SourceId"] = thirdId
                    suningItem["ParentId"] = secondId
                    suningItem['CreatedDate'] = datetime.datetime.now()
                    suningItem['SyncTime'] = datetime.datetime.now()
                    yield suningItem
        pass


#!/usr/bin/env python
import sys
import os
name = ""
base_name = ""

if sys.argv[1] == "":
	print("You need to specify a file to convert")
	sys.exit()
else:
	name = sys.argv[1]

f = open(os.path.dirname(os.path.realpath(__file__)) + "\\" + name, "rb")
fo = open(os.path.dirname(os.path.realpath(__file__)) + "\\" + name + ".r16m", "wb")

data = [int(x) for x in f.read()]

for i in range(0, len(data), 8):
	out = [data[i+1], data[i], data[i+5], data[i+4], 0x00, 0x00, 0x00, 0x00, data[i+3], data[i+2], data[i+7], data[i+6], 0x00, 0x00, 0x00, 0x00]
	fo.write(bytes(out))

fo.close()


import gdb
import re
import subprocess

def get_current_arch():
    arch = gdb.newest_frame().architecture().name()
    #list 'borrowed' from pwndbg
    arches = ['x86-64', 'i386', 'mips', 'powerpc', 'sparc', 'arm', 'aarch64', arch]
    arch = next(a for a in arches if a in arch)
    if arch == "x86-64":
        arch = "amd64"

    return arch

def get_reg(reg):
    reg = gdb.execute("info registers %s" %(reg), to_string=True)
    if reg:
        reg = reg.splitlines()
        if len(reg) > 1:
            return None
        else:
            result = int(reg[0].split()[1],0)
            return result
    return reg

class WAsm(gdb.Command):
    """Write assembler on the fly into memory
wasm                              - Shows context used
wasm <assembler code ... >        - Write at current $pc
wasm <addr> <assembler code ... > - Write at address <addr>
wasm context <context>            - Set context for pwntools asm to <context>. no argument unsets context
    """
    def __init__(self):
        super(WAsm, self).__init__ ("wasm", gdb.COMMAND_USER)
        self.context = None

    def invoke(self, arg, from_tty):
        cmd = arg.split(" ", 1)
        cmd.append(None)

        if arg == "":
            self.show_context()
        elif cmd[0] == "context":
            self.set_context(cmd[1])
        elif re.match("^0x[0-9a-fA-F]+$", cmd[0]):
            self.patch_addr(int(cmd[0], 16), cmd[1])
        else:
            self.patch_pc(arg)

    def show_context(self):
        if self.context == None:
            if gdb.selected_inferior().pid != 0:
                print("Context determined automatically as: {}".format(get_current_arch()))
            else:
                print("No Context set, and not in a running target")
        else:
            print("Context for pwntool asm: {}".format(self.context))

    def set_context(self, context):
        self.context = context

    def patch_pc(self, code):
        if gdb.selected_inferior().pid == 0:
            print("Target is not running, no $pc register available")
            return

        ip = get_reg("pc")
        self.patch_addr(ip, code)

    def patch_addr(self, addr, code):
        inf = gdb.selected_inferior()

        if inf.pid == 0:
            print("Target is not running, there is no memory")
            return

        context = self.context
        if context == None:
            context = get_current_arch()

        cmd = ['pwn', 'asm', '-f','raw', '-c', context, code]
        opcodes = subprocess.check_output(cmd)

        inf.write_memory(addr, opcodes)

WAsm()


import collections
import mlpy
import numpy as np
import scipy as sp
import csv
import string
from sklearn import linear_model

recipesPath = '../mealplan/recipeData.txt'

cuisineMap = {}
cuisineMap['NorthernEuropean'] =250
cuisineMap['WesternEuropean'] =2659
cuisineMap['SouthernEuropean'] =4180
cuisineMap['MiddleEastern'] =645
cuisineMap['SouthAsian'] =621
cuisineMap['LatinAmerican'] =2917
cuisineMap['NorthAmerican'] =41524
cuisineMap['EasternEuropean'] =381
cuisineMap['SoutheastAsian'] =457
cuisineMap['African'] =352
cuisineMap['EastAsian'] =2512

oppMap = {}
for k,v in cuisineMap.iteritems():
	oppMap[v] = k

def process(filename):
	ingredientSet = getIngreds()
	X = []
	Y = []
	ingredientsList = []
	countCuisine = 0
	with open(filename, 'rb') as dataset:
		datareader = csv.reader(dataset)
		tempX = []
		for row in datareader:
			cuisine = row[0]
			orig = row[1:]
			ingredients = collections.Counter(orig)
			ingredientsList.append(orig)
			tempX.append((ingredients, cuisineMap.get(cuisine)))
		for ingredients, cuisine in tempX:
			featureVector = [0] * len(ingredientSet)
			for index, ingredient in enumerate(ingredientSet):
				if ingredient in ingredients:
					featureVector[index] = 1
				else:
					featureVector[index] = 0
			X.append(featureVector)
			Y.append(cuisine)
		return X, Y, ingredientSet, cuisineMap, ingredientsList

def readIngredients(predictor, ingredientSet):
	ingredientsList = collections.OrderedDict()
	with open(recipesPath, 'rb') as dataset, open('./recipeWithCuisines.txt', 'wb') as output:
		for line in dataset:
			orig = line.strip()
			ingredients = []
			line = line.split("<>")
			ingredString = line[6]
			ingredList = ingredString.split(',')
			for ingred in ingredList:
				ingred = ingred.split(';')
				ingredients.append(ingred[0])
			testX = constructFeatureVector(ingredients, ingredientSet)
			cuisine = predictor.predict([testX])
			output.write(orig + "<>" + oppMap[cuisine[0]] + '\n')
	return ingredientsList

def constructFeatureVector(ingredients, ingredientSet):
	featureVector = [0] * len(ingredientSet)
	for index, ingredient in enumerate(ingredientSet):
		if ingredient in ingredients:
			featureVector[index] = 1
		else:
			featureVector[index] = 0
	return featureVector

def getIngreds(file1='ingredients_stats'):
	ingredientSet = set()
	with open(file1, 'rb') as f:
		for line in f:
			ingredientSet.add(line.split()[0])
	return ingredientSet

def predictCuisine(trainX, trainY):
	logreg = linear_model.LogisticRegression()
	logreg.fit(trainX, trainY)
	return logreg

trainX, trainY, ingredientSet, cuisineMap, trainIngredientsList = process("./data.csv")
predictor = predictCuisine(trainX, trainY)
recipeIngredientsList = readIngredients(predictor, ingredientSet)


n = int(input())
a = list(map(int, input().split()))
ans = 10**10
for i in range(n):
    cnt = 0
    while a[i]%2 == 0:
        cnt += 1
        a[i] //= 2
    ans = min(ans, cnt)
print(ans)


from flask_restx import fields

from ...restplus import api
from ...m4i_generic_entity_serializers import m4i_generic_entity_model

m4i_kubernetes_service_process_model = api.inherit('model_m4i_kubernetes_service_process', m4i_generic_entity_model, {
    'processOwner': fields.String(required=False, description='Qualified Name of processOwner of Process Entity'),
    'inputs': fields.List(fields.String(), required=True,
                          description='List Qualified Name Inputs  of Kubernetes Service Process Entity'),
    'outputs': fields.List(fields.String(), required=True,
                           description='List Qualified Name Outputs of Kubernetes Service Entity'),
    'microservice': fields.List(fields.String(), required=True,
                                description='List Qualified Name microservices known by the Kubernetes Service Entity'),
    'namespace': fields.String(required=True, description='The namespace that the Kubernetes Service is on'),
    'ingressObject': fields.String(required=False, description='The ingress object that knows the Kubernetes Service')

})


import time
import functools
import numpy as np

def my_decorator(stats = False):
    times = []
    def decorator(func):
        def wrapper(*args, **kwargs):
            start = time.time()
            value = func(*args,**kwargs)
            stop = time.time()
            times.append(stop-start)
            if stats == True:
                suma = 0
                for t in times:
                    suma+=t
                mean = suma/len(times)
                print("Mean: ",mean)
                print("Minumum: ",min(times))
                print("Maximum: ",max(times))
                print("Standard deviation: ",np.std(times))
            return value

        return wrapper

    return decorator

@my_decorator(True)
def useless(n):
    for i in range(1000000):
        if i%3==2:
            n=n*-0.5**9
        if i%5==0:
            n=n+n/100**3
        for i in range(5):
            n+=1
    return n

print(useless(1))
print(useless(-20))
print(useless(35))
print(useless(-2000))
print(useless(350))


from unittest.mock import patch

import pytest
from airflow.exceptions import AirflowException

from catalog.tests.dags.providers.provider_api_scripts.resources.json_load import (
    make_resource_json_func,
)
from providers.provider_api_scripts.smithsonian import SmithsonianDataIngester

# Set up test class
ingester = SmithsonianDataIngester()
_get_resource_json = make_resource_json_func("smithsonian")

def test_get_hash_prefixes_with_len_one():
    with patch.object(ingester, "hash_prefix_length", 1):
        actual_prefix_list = list(ingester._get_hash_prefixes())
    expect_prefix_list = [
        "0",
        "1",
        "2",
        "3",
        "4",
        "5",
        "6",
        "7",
        "8",
        "9",
        "a",
        "b",
        "c",
        "d",
        "e",
        "f",
    ]
    assert actual_prefix_list == expect_prefix_list

def test_alert_new_unit_codes():
    sub_prov_dict = {"sub_prov1": {"a", "c"}, "sub_prov2": {"b"}, "sub_prov3": {"e"}}
    unit_code_set = {"a", "b", "c", "d"}
    with patch.dict(ingester.sub_providers, sub_prov_dict, clear=True):
        actual_codes = ingester._get_new_and_outdated_unit_codes(unit_code_set)
    expected_codes = ({"d"}, {"e"})
    assert actual_codes == expected_codes

@pytest.mark.parametrize(
    "new_unit_codes, outdated_unit_codes",
    [
        ({"d"}, {"e"}),
        ({"d"}, set()),
        (set(), {"e"}),
    ],
)
def test_validate_unit_codes_from_api_raises_exception(
    new_unit_codes, outdated_unit_codes
):
    with patch.object(ingester, "_get_unit_codes_from_api"), patch.object(
        ingester,
        "_get_new_and_outdated_unit_codes",
        return_value=(new_unit_codes, outdated_unit_codes),
    ):
        message = "^\n\\*Updates needed to the SMITHSONIAN_SUB_PROVIDERS dictionary\\**"
        with pytest.raises(AirflowException, match=message):
            ingester.validate_unit_codes_from_api()

def test_validate_unit_codes_from_api():
    with patch.object(ingester, "_get_unit_codes_from_api"), patch.object(
        ingester, "_get_new_and_outdated_unit_codes", return_value=(set(), set())
    ):
        # Validation should run without raising an exception
        ingester.validate_unit_codes_from_api()

@pytest.mark.parametrize(
    "input_int, expect_len, expect_first, expect_last",
    [
        (1, 16, "0", "f"),
        (2, 256, "00", "ff"),
        (3, 4096, "000", "fff"),
        (4, 65536, "0000", "ffff"),
    ],
)
def test_get_hash_prefixes_with_other_len(
    input_int, expect_len, expect_first, expect_last
):
    with patch.object(ingester, "hash_prefix_length", input_int):
        actual_list = list(ingester._get_hash_prefixes())

    assert all("0x" not in h for h in actual_list)
    assert all(
        int(actual_list[i + 1], 16) - int(actual_list[i], 16) == 1
        for i in range(len(actual_list) - 1)
    )
    assert len(actual_list) == expect_len
    assert actual_list[0] == expect_first
    assert actual_list[-1] == expect_last

def test_get_next_query_params_first_call():
    hash_prefix = "ff"
    actual_params = ingester.get_next_query_params(
        prev_query_params=None, hash_prefix=hash_prefix
    )
    actual_params.pop("api_key")  # Omitting the API key
    expected_params = {
        "q": f"online_media_type:Images AND media_usage:CC0 AND hash:{hash_prefix}*",
        "start": 0,
        "rows": 1000,
    }
    assert actual_params == expected_params

def test_get_next_query_params_updates_parameters():
    previous_query_params = {
        "api_key": "pass123",
        "q": "online_media_type:Images AND media_usage:CC0 AND hash:ff*",
        "start": 0,
        "rows": 1000,
    }
    actual_params = ingester.get_next_query_params(previous_query_params)
    expected_params = {
        "api_key": "pass123",
        "q": "online_media_type:Images AND media_usage:CC0 AND hash:ff*",
        "start": 1000,
        "rows": 1000,
    }
    assert actual_params == expected_params

@pytest.mark.parametrize(
    "input_row, field, expected_dict",
    [
        # field: descriptiveNonRepeating
        ({}, "descriptiveNonRepeating", {}),
        ({"content": {"key": {"key2", "val2"}}}, "descriptiveNonRepeating", {}),
        (
            {"noncontent": {"descriptiveNonRepeating": {"key2": "val2"}}},
            "descriptiveNonRepeating",
            {},
        ),
        (
            {"content": {"descriptiveNonRepeating": {"key2": "val2"}}},
            "descriptiveNonRepeating",
            {"key2": "val2"},
        ),
        # field: indexedStructured
        ({}, "indexedStructured", {}),
        ({"content": {"key": {"key2", "val2"}}}, "indexedStructured", {}),
        (
            {"noncontent": {"indexedStructured": {"key2": "val2"}}},
            "indexedStructured",
            {},
        ),
        (
            {"content": {"indexedStructured": {"key2": "val2"}}},
            "indexedStructured",
            {"key2": "val2"},
        ),
        # field: freetext
        ({}, "freetext", {}),
        ({"content": {"key": {"key2", "val2"}}}, "freetext", {}),
        ({"noncontent": {"freetext": {"key2": "val2"}}}, "freetext", {}),
        ({"content": {"freetext": {"key2": "val2"}}}, "freetext", {"key2": "val2"}),
    ],
)
def test_content_dict(input_row, field, expected_dict):
    actual_dict = ingester._get_content_dict(input_row, field)
    assert actual_dict == expected_dict

@pytest.mark.parametrize(
    "required_type,default_input",
    [
        (str, ""),
        (int, 0),
        (float, 0.0),
        (complex, 0j),
        (list, []),
        (tuple, ()),
        (dict, {}),
        (set, set()),
        (bool, False),
        (bytes, b""),
    ],
)
def test_check_type_with_defaults(required_type, default_input):
    assert ingester._check_type(default_input, required_type) == default_input

@pytest.mark.parametrize(
    "required_type,good_input",
    [
        (str, "abc"),
        (int, 5),
        (float, 1.2),
        (complex, 3 + 2j),
        (list, ["a", 2, 0, False]),
        (tuple, (1, 0, False)),
        (dict, {"key": "val", "f": False}),
        (set, {1, False, "abc"}),
        (bool, True),
        (bytes, b"abc"),
    ],
)
def test_check_type_with_truthy_good_inputs(required_type, good_input):
    assert ingester._check_type(good_input, required_type) == good_input

@pytest.mark.parametrize(
    "required_type,good_indices,default",
    [
        (str, (0, 1), ""),
        (int, (2, 3), 0),
        (float, (4, 5), 0.0),
        (complex, (6, 7), 0j),
        (list, (8, 9), []),
        (tuple, (10, 11), ()),
        (dict, (12, 13), {}),
        (set, (14, 15), set()),
        (bool, (16, 17), False),
        (bytes, (18, 19), b""),
    ],
)
def test_check_type_with_bad_inputs(required_type, good_indices, default):
    bad_input_list = [
        "abc",
        "",
        5,
        0,
        1.2,
        0.0,
        3 + 2j,
        0j,
        ["a", 2],
        [],
        (1, 2),
        (),
        {"key": "val"},
        {},
        {1, "abc"},
        set(),
        True,
        False,
        b"abc",
        b"",
    ]
    for i in sorted(good_indices, reverse=True):
        del bad_input_list[i]
    for bad_input in bad_input_list:
        assert ingester._check_type(bad_input, required_type) == default

@pytest.mark.parametrize(
    "input_dnr, expect_image_list",
    [
        ({}, []),
        ({"non_media": {"media": ["image1", "image2"]}}, []),
        ({"online_media": "wrong type"}, []),
        ({"online_media": ["wrong", "type"]}, []),
        ({"online_media": {"media": "wrong type"}}, []),
        ({"online_media": {"media": {"wrong": "type"}}}, []),
        (
            {
                "record_ID": "siris_arc_291918",
                "online_media": {"mediaCount": 1, "media": ["image1", "image2"]},
            },
            ["image1", "image2"],
        ),
    ],
)
def test_get_image_list(input_dnr, expect_image_list):
    input_row = {"key": "val"}
    with patch.object(
        ingester, "_get_content_dict", return_value=input_dnr
    ) as mock_dnr:
        actual_image_list = ingester._get_image_list(input_row)
    mock_dnr.assert_called_once_with(input_row, "descriptiveNonRepeating")
    assert actual_image_list == expect_image_list

def test_get_media_type():
    actual_type = ingester.get_media_type(record={})
    assert actual_type == "image"

@pytest.mark.parametrize(
    "input_dnr, expect_foreign_landing_url",
    [
        ({}, None),
        (
            {
                "guid": "http://fallback.com",
                "unit_code": "NMNHMAMMALS",
                "record_link": "http://chooseme.com",
            },
            "http://chooseme.com",
        ),
        ({"guid": "http://fallback.com"}, "http://fallback.com"),
        ({"no": "urlhere"}, None),
    ],
)
def test_get_foreign_landing_url(input_dnr, expect_foreign_landing_url):
    input_row = {"key": "val"}
    with patch.object(
        ingester, "_get_content_dict", return_value=input_dnr
    ) as mock_dnr:
        actual_foreign_landing_url = ingester._get_foreign_landing_url(input_row)
    mock_dnr.assert_called_once_with(input_row, "descriptiveNonRepeating")
    assert actual_foreign_landing_url == expect_foreign_landing_url

@pytest.mark.parametrize(
    "input_is, input_ft, expected_creator",
    [
        ({}, {}, None),
        ({"name": ["Alice"]}, {}, None),
        ({"name": "alice"}, {}, None),
        ({"name": [{"type": ["personal", "main"], "nocontent": "Alice"}]}, {}, None),
        ({"name": [{"type": "personal_main", "nocontent": "Alice"}]}, {}, None),
        ({"noname": [{"type": "personal_main", "content": "Alice"}]}, {}, None),
        ({"name": [{"label": "personal_main", "content": "Alice"}]}, {}, None),
        ({"name": [{"type": "impersonal_main", "content": "Alice"}]}, {}, None),
        (
            {"name": [{"type": "personal_main", "content": "Alice"}]},
            {"name": "Bob"},
            "Alice",
        ),
        (
            {"name": [{"type": "personal_main", "content": "Alice"}]},
            {"name": ["Bob"]},
            "Alice",
        ),
        (
            {"name": [{"type": "personal_main", "content": "Alice"}]},
            {"name": [{"label": "Creator", "nocontent": "Bob"}]},
            "Alice",
        ),
        (
            {"name": [{"type": "personal_main", "content": "Alice"}]},
            {"name": [{"nolabel": "Creator", "content": "Bob"}]},
            "Alice",
        ),
        (
            {"name": [{"type": "personal_main", "content": "Alice"}]},
            {"name": [{"label": "NotaCreator", "content": "Bob"}]},
            "Alice",
        ),
        (
            {"name": [{"type": "personal_main", "content": "Alice"}]},
            {"noname": [{"label": "Creator", "content": "Bob"}]},
            "Alice",
        ),
        (
            {"name": [{"type": "personal_main", "content": "Alice"}]},
            {"name": [{"label": "Creator", "content": "Bob"}]},
            "Bob",
        ),
        (
            {},
            {
                "name": [
                    {"label": "Designer", "content": "Alice"},
                    {"label": "Creator", "content": "Bob"},
                ]
            },
            "Bob",
        ),
        (
            {},
            {
                "name": [
                    {"label": "AFTER", "content": "Bob"},
                    {"label": "Designer", "content": "Alice"},
                ]
            },
            "Alice",
        ),
        (
            {},
            {
                "name": [
                    {"label": "AFTER", "content": "Bob"},
                    {"label": "DESIGNER", "content": "Alice"},
                ]
            },
            "Alice",
        ),
        (
            {
                "name": [
                    {"type": "personal_main", "content": "Alice"},
                    {"type": "corporate_subj", "content": "Zoological Park"},
                ]
            },
            {
                "name": [
                    {"label": "Creator", "content": "Bob"},
                    {"label": "Subject", "content": "Zoological Park"},
                ]
            },
            "Bob",
        ),
    ],
)
def test_get_creator(input_is, input_ft, expected_creator):
    creator_types = {"creator": 0, "designer": 1, "after": 3}
    input_row = {
        "test": "row",
        "content": {"freetext": input_ft, "indexedStructured": input_is},
    }
    with patch.object(ingester, "creator_types", creator_types):
        actual_creator = ingester._get_creator(input_row)

    assert actual_creator == expected_creator

@pytest.mark.parametrize(
    "input_ft, input_dnr, expected_description",
    [
        ({}, {}, None),
        ({"notes": [{"label": "notthis", "content": "blah"}]}, {}, None),
        ({"notes": "notalist"}, {}, None),
        ({"notes": [{"label": "Summary", "content": "blah"}]}, {}, "blah"),
        (
            {
                "notes": [
                    {"label": "Description", "content": "blah"},
                    {"label": "Summary", "content": "blah"},
                    {"label": "Description", "content": "blah"},
                ]
            },
            {},
            "blah blah blah",
        ),
        (
            {
                "notes": [
                    {"label": "notDescription", "content": "blah"},
                    {"label": "Summary", "content": "blah"},
                    {"label": "Description", "content": "blah"},
                ]
            },
            {},
            "blah blah",
        ),
    ],
)
def test_ext_meta_data_description(input_ft, input_dnr, expected_description):
    description_types = {"description", "summary"}
    input_row = {
        "test": "row",
        "content": {"freetext": input_ft, "descriptiveNonRepeating": input_dnr},
    }
    with patch.object(ingester, "description_types", description_types):
        meta_data = ingester._extract_meta_data(input_row)
    actual_description = meta_data.get("description")

    assert actual_description == expected_description

@pytest.mark.parametrize(
    "input_ft, input_dnr, expected_label_text",
    [
        ({}, {}, None),
        ({"notes": [{"label": "notthis", "content": "blah"}]}, {}, None),
        ({"notes": "notalist"}, {}, None),
        ({"notes": [{"label": "Label Text", "content": "blah"}]}, {}, "blah"),
        (
            {
                "notes": [
                    {"label": "Label Text", "content": "blah"},
                    {"label": "Summary", "content": "halb"},
                    {"label": "Description", "content": "halb"},
                ]
            },
            {},
            "blah",
        ),
    ],
)
def test_ext_meta_data_label_text(input_ft, input_dnr, expected_label_text):
    input_row = {
        "test": "row",
        "content": {"freetext": input_ft, "descriptiveNonRepeating": input_dnr},
    }
    meta_data = ingester._extract_meta_data(input_row)
    actual_label_text = meta_data.get("label_text")

    assert actual_label_text == expected_label_text

@pytest.mark.parametrize(
    "input_ft, input_dnr, expected_meta_data",
    [
        ({"nothing": "here"}, {"nothing_to": "see"}, {}),
        ({}, {"unit_code": "SIA"}, {"unit_code": "SIA"}),
        (
            {},
            {"data_source": "Smithsonian Institution Archives"},
            {"data_source": "Smithsonian Institution Archives"},
        ),
    ],
)
def test_extract_meta_data_dnr_fields(input_ft, input_dnr, expected_meta_data):
    input_row = {
        "test": "row",
        "content": {"freetext": input_ft, "descriptiveNonRepeating": input_dnr},
    }
    actual_meta_data = ingester._extract_meta_data(input_row)

    assert actual_meta_data == expected_meta_data

@pytest.mark.parametrize(
    "input_is, expect_tags",
    [
        ({}, []),
        ({"nothing": "here"}, []),
        (
            {
                "date": ["", ""],
                "place": ["Indian Ocean"],
            },
            ["Indian Ocean"],
        ),
        (
            {
                "date": ["2000s"],
                "object_type": ["Holotypes", "Taxonomic type specimens"],
                "topic": ["Paleogeneral", "Protists"],
                "place": ["Indian Ocean"],
            },
            [
                "2000s",
                "Holotypes",
                "Taxonomic type specimens",
                "Paleogeneral",
                "Protists",
                "Indian Ocean",
            ],
        ),
    ],
)
def test_extract_tags(input_is, expect_tags):
    input_row = {"test": "row"}
    get_is = patch.object(ingester, "_get_content_dict", return_value=input_is)
    with get_is as mock_is:
        actual_tags = ingester._extract_tags(input_row)
    mock_is.assert_called_once_with(input_row, "indexedStructured")
    assert actual_tags == expect_tags

@pytest.mark.parametrize(
    "input_media, expected_image_data",
    [
        ([], []),
        (
            [
                {
                    "thumbnail": "https://thumbnail.one",
                    "idsId": "id_one",
                    "usage": {"access": "CC0"},
                    "guid": "http://gu.id.one",
                    "type": "Images",
                    "content": "",
                },
            ],
            [],
        ),
        (
            [
                {
                    "thumbnail": "https://thumbnail.one",
                    "idsId": "",
                    "usage": {"access": "CC0"},
                    "guid": "http://gu.id.one",
                    "type": "Images",
                    "content": "https://image.url.one",
                }
            ],
            [],
        ),
        (
            [
                {
                    "thumbnail": "https://thumbnail.one",
                    "idsId": "id_one",
                    "usage": {"access": "CC0"},
                    "guid": "http://gu.id.one",
                    "type": "Images",
                    "content": "https://image.url.one",
                },
                {
                    "thumbnail": "https://thumbnail.two",
                    "idsId": "id_two",
                    "usage": {"access": "CC0"},
                    "guid": "http://gu.id.two",
                    "type": "Images",
                    "content": "https://image.url.two",
                },
            ],
            [
                {
                    "foreign_identifier": "id_one",
                    "url": "https://image.url.one",
                },
                {
                    "foreign_identifier": "id_two",
                    "url": "https://image.url.two",
                },
            ],
        ),
    ],
)
def test_process_image_list(input_media, expected_image_data):
    shared_image_data = {
        "foreign_landing_url": "https://foreignlanding.url",
        "title": "The Title",
        "license_info": ingester.license_info,
        "creator": "Alice",
        "meta_data": {"unit_code": "NMNHBOTANY"},
        "source": "smithsonian_national_museum_of_natural_history",
        "raw_tags": ["tag", "list"],
    }
    expected_result = [(img | shared_image_data) for img in expected_image_data]
    actual_image_data = ingester._get_associated_images(input_media, shared_image_data)
    assert actual_image_data == expected_result

def test_get_record_data():
    data = _get_resource_json("actual_record_data.json")
    actual_data = ingester.get_record_data(data)
    expected_data = [
        {
            "url": "https://collections.nmnh.si.edu/media/?irn=15814382",
            "foreign_identifier": "https://collections.nmnh.si.edu/media/?irn=15814382",
            "foreign_landing_url": "http://n2t.net/ark:/65665/34857ca78-9195-4156-849b-1ec47f7cd1ce",
            "title": "Passerculus sandwichensis nevadensis",
            "license_info": ingester.license_info,
            "source": "smithsonian_national_museum_of_natural_history",
            "creator": "Seymour H. Levy",
            "meta_data": {
                "unit_code": "NMNHBIRDS",
                "data_source": "NMNH - Vertebrate Zoology - Birds Division",
            },
            "raw_tags": [
                "1950s",
                "Animals",
                "Birds",
                "United States",
                "Pinal",
                "North America",
                "Arizona",
            ],
        }
    ]

    assert actual_data == expected_data


import os
import json

from datatiles.encoding import encode_tifs
from datatiles.mbtiles import tif_to_mbtiles, render_tif_to_mbtiles

max_zoom = 15
datatiles_tile_size = (
    128
)  # Note: this must be set into 'imageSize' in the corresponding Leaflet.DataTileLayer in the frontend
data_dir = "/Volumes/data/projects/SA_LCC/Conservation_Blueprint_2.2/data/source/"
tile_dir = "./tiles"

# visible Blueprint layer
render_tif_to_mbtiles(
    os.path.join(data_dir, "Blueprint_2_2.tif"),
    os.path.join(tile_dir, "blueprint2_2.mbtiles"),
    # Colors are the standard Blueprint 2.2 colors, in /config/priorities.json
    colormap={1: "#004DA8", 2: "#686868", 3: "#fbb4b9", 4: "#c51b8a", 5: "#49006a"},
    min_zoom=0,
    max_zoom=max_zoom,
    metadata={
        "name": "South Atlantic Conservation Blueprint v2.2",
        "description": "South Atlantic Conservation Blueprint v2.2",
        "attribution": "South Atlantic Blueprint 2018",
    },
)

# Data tiles (blueprint & indicators)
# Note: the encoding JSON that results from running the following is pasted into /config/encoding.json

sources = {
    "2": {
        # Binary: presence / absence or nodata (3 values)
        "ForestedWetland_Amphibians": {
            "source": os.path.join(data_dir, "ForestedWetland_Amphibians_V_2_1Clip.tif")
        },
        "ForestedWetland_Extent": {
            "source": os.path.join(
                data_dir,
                "ForestedWetland_Extent_V_2_2_NoDataTo0_ClipToEcosystemMaskClip.tif",
            )
        },
        "FreshwaterMarsh_Extent": {
            "source": os.path.join(
                data_dir,
                "FreshwaterMarsh_Extent_V_2_2_NoDataTo0_ClipToEcosystemMaskClip.tif",
            )
        },
        "Landscapes_LowRoadDensityPatches": {
            "source": os.path.join(
                data_dir, "Landscapes_LowRoadDensityPatches_V_2_1Clip.tif"
            )
        },
        "Landscapes_LowUrbanHistoric": {
            "source": os.path.join(
                data_dir, "Landscapes_LowUrbanHistoric_V_2_1_SOTClip.tif"
            )
        },
        "MaritimeForest_Extent": {
            "source": os.path.join(
                data_dir,
                "MaritimeForest_Extent_V_2_0_NoDataTo0_ClipToEcosystemMaskClip.tif",
            )
        },
        "PineAndPrairie_Amphibians": {
            "source": os.path.join(data_dir, "PineAndPrairie_Amphibians_V_2_1Clip.tif")
        },
        "PineAndPrairie_RegularlyBurnedHabitat": {
            "source": os.path.join(
                data_dir, "PineAndPrairie_RegularlyBurnedHabitat_V_2_0Clip.tif"
            )
        },
    },
    "3": {
        # 3 values
        "BeachAndDune_UnalteredBeach": {
            "source": os.path.join(
                data_dir, "BeachAndDune_UnalteredBeach_V_2_2Clip.tif"
            )
        },
        "FreshwaterMarsh_Birds": {
            "source": os.path.join(
                data_dir, "FreshwaterMarsh_Birds_V_2_1_BinnedClip.tif"
            )
        },
        # 4 values
        "ForestedWetland_Birds": {
            "source": os.path.join(data_dir, "ForestedWetland_Birds_V_2_1Clip.tif")
        },
        "FreshwaterAquatic_PermeableSurface": {
            "source": os.path.join(
                data_dir, "FreshwaterAquatic_PermeableSurface_V_2_1_BinnedClip.tif"
            )
        },
        "PineAndPrairie_Birds": {
            "source": os.path.join(data_dir, "PineAndPrairie_Birds_V_2_1Clip.tif")
        },
        "Waterscapes_MigratoryFishConnectivity": {
            "source": os.path.join(
                data_dir, "Waterscapes_MigratoryFishConnectivity_V_2_1Clip.tif"
            )
        },
    },
    "5": {
        "BeachAndDune_BeachBirds": {
            "source": os.path.join(
                data_dir, "BeachAndDune_BeachBirds_V_2_2_BinnedClip.tif"
            )
        },
        "Estuarine_CoastalCondition": {
            "source": os.path.join(
                data_dir, "Estuarine_CoastalCondition_V_2_1_BinnedClip.tif"
            )
        },
        "Estuarine_WaterVegetationEdge": {
            "source": os.path.join(
                data_dir, "EstuarineMarsh_Water_VegetationEdge_V_2_0Clip.tif"
            )
        },
        "Estuarine_WetlandPatchSize": {
            "source": os.path.join(
                data_dir, "EstuarineMarsh_WetlandPatchSize_V_2_0Clip.tif"
            )
        },
        "FreshwaterAquatic_ImperiledAquaticSpecies": {
            "source": os.path.join(
                data_dir, "FreshwaterAquatic_ImperiledAquaticSpecies_V_2_1Clip.tif"
            )
        },
        "FreshwaterAquatic_RiparianBuffers": {
            "source": os.path.join(
                data_dir, "FreshwaterAquatic_RiparianBuffers_V_2_1_BinnedClip.tif"
            )
        },
        "Marine_Birds": {
            "source": os.path.join(data_dir, "Marine_Birds_V_2_2_BinnedClip.tif")
        },
        "Marine_Mammals": {
            "source": os.path.join(data_dir, "Marine_Mammals_V_2_1_BinnedClip.tif")
        },
    },
    "7": {
        # 5 values, spillover from above
        "Marine_PotentialHardbottomCondition": {
            "source": os.path.join(
                data_dir, "Marine_PotentialHardbottomCondition_V_2_0Clip.tif"
            )
        },
        "UplandHardwood_Birds": {
            "source": os.path.join(data_dir, "UplandHardwood_Birds_V_2_0Clip.tif")
        },
        # 7 values
        "Blueprint": {"source": os.path.join(data_dir, "Blueprint_2_2.tif")},
        "UplandHardwood_UrbanOpenSpace": {
            "source": os.path.join(
                data_dir, "UplandHardwood_UrbanOpenSpace_V_2_1Clip.tif"
            )
        },
        # 8 values
        "Waterscapes_NetworkComplexity": {
            "source": os.path.join(
                data_dir, "Waterscapes_NetworkComplexity_V_2_1Clip.tif"
            )
        },
        # 9 values
        "Landscapes_ResilientBiodiversityHotspots": {
            "source": os.path.join(
                data_dir, "Landscapes_ResilientBiodiversityHotspots_V_2_2Clip.tif"
            )
        },
    },
}

for level in sources:
    print("Encoding level {}...".format)
    tmpfilename = "/tmp/encoding{}.tif".format(level)

    # Encode individual tifs to a single tif using exponential encoder
    encoding = encode_tifs(sources[level], tmpfilename, encoding="exponential")

    print("Encoding config is\n\n{}\n\n".format(json.dumps(encoding, indent=4)))

    # Convert the tif to tiles
    tif_to_mbtiles(
        tmpfilename,
        os.path.join(tile_dir, "encoding{}.mbtiles".format(level)),
        max_zoom=max_zoom,
        tile_size=datatiles_tile_size,
    )


'''
1.1 Getting started with Pandas
Learn how to use Pandas Series
Learn how to use Pandas DataFrames
Scotch whiskies based on their flavor characteristic
'''

# Pandas - 2 data structure: Series(1 demensional) and Data Frames (2 dimensional)

import pandas as pd

########
# Series
########
# create object x contain 4 numbers
x = pd.Series([6,3,8,6])
print(x)
#define indexes corresponding to value
x = pd.Series([6,3,8,6], index=["q","w","e","r"])
print(x)

print(x["w"])

print(x[["r","w"]])
#define a dictionary
age = {"Tim":29,"Jim":31,"Pam":27,"Sam":35}

x = pd.Series(age)
print(x)

########
# Data Frame
########

data ={'name': ['Tim', 'Jim', 'Pam', 'Sam'],
        'age': [29, 31, 27, 35],
        'ZIP': ['02115','02130','67700','00100']}

# construct a Data Frames from dictionary
x = pd.DataFrame(data, columns = ["name","age","ZIP"])
print(x)

# retrieve a column
print(x["name"])
print(x.name)

# re-index a Series or a Data Frame object
x = pd.Series([6,3,8,6], index=["q","w","e","r"])
print(x.index)
sorted(x.index)

x.reindex(sorted(x.index))

# add 2 Series object
x = pd.Series([6,3,8,6], index=["q","w","e","r"])
y = pd.Series([7,3,5,2], index=["e","q","r","t"])

print(x + y) # if indexes were different -> NAN in t and w
'''
Index(['q', 'w', 'e', 'r'], dtype='object')
e    15.0
q     9.0
r    11.0
t     NaN
w     NaN
'''
# Pandas can be used to summarize data, compute correlations,
# hanle missing data, use hierachical indexing...

'''
1.2 Loading and inspecting data
Learn how to load a CSV file using Pandas
Learn how to view the beginning and end of a Pandas DataFrame
Learn how to index a Pandas DataFrame by location
'''

import numpy as np
import pandas as pd

whisky = pd.read_csv("/Users/tp10/Sanger_work/Developer101/python_for_research_edXcourse/Week_4/whiskies.txt")

whisky["Region"] = pd.read_csv("/Users/tp10/Sanger_work/Developer101/python_for_research_edXcourse/Week_4/regions.txt")

whisky.head()
whisky.tail()

whisky.iloc[0:10] # row 0 to 10

whisky.iloc[5:10, 0:5] # row 5 to 10 and column 0 to 5

whisky.columns

flavors = whisky.iloc[:, 2:14] # include column 13

print(flavors)

'''
1.3 Exploring Correlations
Learn how to explore correlations in your data
Learn how to plot a correlation matrix
'''
# Pearson correlation for linear data
corr_flavors = pd.DataFrame.corr(flavors)
print(corr_flavors)

import matplotlib.pyplot as plt
plt.figure(figsize=(10,10))
plt.pcolor(corr_flavors)
plt.colorbar()
plt.savefig("corr_flavors.pdf")

#heavy body is associated with smokiness
#a floral flavor is the opposite of full body

'''
Whiskies are made by different distilleries,
so this is correlation bt different distilleries in terms of the flavor
profiles of the whiskies that they produce
'''
corr_whisky = pd.DataFrame.corr(flavors.transpose())
plt.figure(figsize=(10,10))
plt.pcolor(corr_whisky)
plt.colorbar()
plt.savefig("corr_whisky.pdf")

#90 by 90 correlation matrix, but we only has 86x86
# modify the axis to 86 by "tight"
corr_whisky = pd.DataFrame.corr(flavors.transpose())
plt.figure(figsize=(10,10))
plt.pcolor(corr_whisky)
plt.axis("tight")
plt.colorbar()
plt.savefig("corr_whisky_tight.pdf")

'''
1.4 Clustering Whiskies by Flavor Profile
Learn how to use spectral co-clustering to cluster whiskies based on their flavor profiles
'''
# using spectral co-clustering method from scikit-learn machine learning module
# goal is to find a sets of words and sets of documents that often go together
# each cluster consist of both words and documents

from sklearn.cluster.bicluster import SpectralCoclustering

model = SpectralCoclustering(n_clusters=6, random_state=0)
model.fit(corr_whisky)

print(model.rows_)

np.sum(model.rows_, axis = 1)
np.sum(model.rows_, axis = 0)

# 6 clusters (from 0 to 5)
print(model.row_labels_)

'''
1.5 Comparing Correlation Matrices
Learn how to compare correlation matrices
'''
whisky['Group'] = pd.Series(model.row_labels_, index=whisky.index)

whisky = whisky.ix[np.argsort(model.row_labels_)]
whisky = whisky.reset_index(drop=True)

correlations = pd.DataFrame.corr(whisky.iloc[:,2:14].transpose())

correlations = np.array(correlations)

plt.figure(figsize=(14,7))
plt.subplot(121)
plt.pcolor(corr_whisky)
plt.title("Original")
plt.axis("tight")
plt.subplot(122)
plt.pcolor(correlations)
plt.title("Rearranged")
plt.axis("tight")
plt.savefig("correlations.pdf")

import pandas as pd
data = pd.Series([1,2,3,4])
print(data)
data = data.ix[[3,0,1,2]]
print(data)
print(data[0])

data = pd.Series([1,2,3,4])
print(data)
data = data.ix[[3,0,1,2]]
print(data)
data = data.reset_index(drop=True)
print(data)
print(data[0])


from PySide6.QtWidgets import (
    QFrame,
    QGridLayout,
    QLabel,
    QLineEdit,
    QRadioButton,
    QCheckBox,
    QPushButton,
    QWidget,
)
from PySide6.QtGui import QIcon
from PySide6.QtCore import Signal
from utils import validators

class BackgroundRemoval(QFrame):
    """
    A widget for parameters selection for manual background removal.
    """

    # custom signals
    math_morpho_toggled = Signal(bool)
    bubblefill_toggled = Signal(bool)
    poly_deg_changed = Signal(int)
    water_bubble_size_changed = Signal(int)
    bubble_size_changed = Signal(int)
    ignore_water_band_toggled = Signal(bool)
    apply_clicked = Signal()

    def __init__(self, parent: QWidget = None) -> None:
        """
        The constructor for manual bg subtraction parameters selection widget.

        Parameters:
            parent (QWidget): Parent widget of this widget. Default: None.
        """

        super().__init__(parent)

        self.setObjectName("method_instance")
        self.icon = QIcon("src/resources/icons/background.svg")

        self.poly_fit_btn = QRadioButton("Poly-fit (I-ModPoly)")
        self.poly_fit_btn.setChecked(True)

        self.bubblefill_btn = QRadioButton("BubbleFill")
        self.bubblefill_btn.toggled.connect(self.emit_bubblefill_toggled)

        self.math_morpho_btn = QRadioButton("Mathematical Morphology")
        self.math_morpho_btn.toggled.connect(self.emit_math_morpho_toggled)

        self.init_water_bubble_size = 700
        self.init_bubble_size = 100

        self.bubble_range = (1, 1000)

        self.water_bubble_size = QLineEdit(
            str(self.init_water_bubble_size), validator=validators.INT_VALIDATOR
        )
        self.water_bubble_size.editingFinished.connect(self.validate_bubble_size_range)
        self.water_bubble_size.editingFinished.connect(
            self.emit_water_bubble_size_changed
        )

        self.bubble_size = QLineEdit(
            str(self.init_bubble_size), validator=validators.INT_VALIDATOR
        )
        self.bubble_size.editingFinished.connect(self.validate_bubble_size_range)
        self.bubble_size.editingFinished.connect(self.emit_bubble_size_changed)

        self.init_poly_deg = 5
        # NOTE: range is inclusive on both sides
        self.poly_deg_range = (1, 15)
        self.poly_deg = QLineEdit(
            str(self.init_poly_deg), validator=validators.INT_VALIDATOR
        )
        self.poly_deg.editingFinished.connect(self.validate_poly_deg_range)
        self.poly_deg.editingFinished.connect(self.emit_poly_deg_changed)

        self.ignore_water_band = QCheckBox()
        self.ignore_water_band.toggled.connect(self.emit_ignore_water_band)
        self.ignore_water_band.setChecked(True)

        self.apply_button = QPushButton("Apply")
        self.apply_button.clicked.connect(self.apply_clicked.emit)

        # put widgets into layout
        layout = QGridLayout()

        layout.addWidget(QLabel("Ignore Water Band"), 0, 0)
        layout.addWidget(self.ignore_water_band, 0, 1)

        layout.addWidget(self.poly_fit_btn, 1, 0)

        layout.addWidget(QLabel("Polynom Degree"), 2, 0)
        layout.addWidget(self.poly_deg, 2, 1)

        layout.addWidget(self.math_morpho_btn, 3, 0)

        layout.addWidget(self.bubblefill_btn, 4, 0)
        layout.addWidget(QLabel("Bubble Sizes (non-water | water)"), 5, 0)
        layout.addWidget(self.water_bubble_size, 5, 1)
        layout.addWidget(self.bubble_size, 5, 2)

        layout.addWidget(self.apply_button, 6, 3)

        layout.setColumnStretch(2, 1)

        self.setLayout(layout)

    def emit_math_morpho_toggled(self) -> None:
        """
        Handler for `math_morpho_toggled` signal emitting.
        """

        is_checked = self.math_morpho_btn.isChecked()

        # disable input into `poly_deg`
        self.poly_deg.setEnabled(not is_checked)
        self.water_bubble_size.setEnabled(not is_checked)
        self.bubble_size.setEnabled(not is_checked)

        # emit whether math_morpho button is checked
        self.math_morpho_toggled.emit(is_checked)

    def emit_bubblefill_toggled(self) -> None:
        """
        Handler for `bubblefill_toggled` signal emitting.
        """

        is_checked = self.bubblefill_btn.isChecked()

        # disable input into `poly_deg`
        self.poly_deg.setEnabled(not is_checked)
        self.water_bubble_size.setEnabled(is_checked)
        self.bubble_size.setEnabled(is_checked)

        # emit whether bubblefill button is checked
        self.bubblefill_toggled.emit(is_checked)

    def validate_poly_deg_range(self) -> None:
        """
        The function to validate range of `poly_deg` input, setting it to one of the bounds
        if it's invalid.
        """

        poly_deg = int(self.poly_deg.text())
        if poly_deg < self.poly_deg_range[0]:
            self.poly_deg.setText(str(self.poly_deg_range[0]))
        elif poly_deg > self.poly_deg_range[1]:
            self.poly_deg.setText(str(self.poly_deg_range[1]))

    def emit_poly_deg_changed(self) -> None:
        """
        Handler for `poly_deg_changed` signal emitting.
        """

        poly_deg_value = int(self.poly_deg.text())

        # emit current value of `self.poly_deg`
        self.poly_deg_changed.emit(poly_deg_value)

    def emit_ignore_water_band(self) -> None:
        """
        Handler for `ignore_water_band_toggled` signal emitting.
        """

        # emit whether `self.ignore_water_band` is checked right now
        self.ignore_water_band_toggled.emit(self.ignore_water_band.isChecked())

    def validate_bubble_size_range(self) -> None:
        bubbles = [self.bubble_size, self.water_bubble_size]

        for bubble in bubbles:
            bubble_size = int(bubble.text())
            if bubble_size < self.bubble_range[0]:
                bubble.setText(str(self.bubble_range[0]))
            elif bubble_size > self.bubble_range[1]:
                bubble.setText(str(self.bubble_range[1]))

    def emit_water_bubble_size_changed(self) -> None:
        """
        Handler for `water_bubble_size_changed` signal emitting.
        """

        bubble_size = int(self.water_bubble_size.text())
        self.water_bubble_size_changed.emit(bubble_size)

    def emit_bubble_size_changed(self) -> None:
        """
        Handler for `bubble_size_changed` signal emitting.
        """
        bubble_size = int(self.bubble_size.text())
        self.bubble_size_changed.emit(bubble_size)

    def get_params(self) -> tuple[int, bool]:
        """
        The function to get parameters from all inputs.

        Returns:
            parameters (tuple): Tuple of background removal method parameters converted to correct types.
        """

        parameters = (
            int(self.poly_deg.text()),
            self.ignore_water_band.isChecked(),
            int(self.bubble_size.text()),
            int(self.water_bubble_size.text()),
        )
        return parameters

    def reset(self) -> None:
        """
        The function to reset all widgets to initial state.
        """

        self.poly_fit_btn.setChecked(True)
        self.ignore_water_band.setChecked(True)
        self.poly_deg.setText(str(self.init_poly_deg))
        self.water_bubble_size.setText(str(self.init_water_bubble_size))
        self.bubble_size.setText(str(self.init_bubble_size))

    def get_string_name(self) -> str:
        """
        A function to return name of this widget as a string.

        Returns:
            widget_name (str): Name of the widget so that it can be recognized by the user.
        """

        return "Background Removal"


#get the set of characters in a language

meta = '/data/mhcyone/onespeaker.csv'

f = open(meta,'r')
t = f.read()
f.close()

t = t.split('\n')

chars = set()
for line in t:
	bits = line.split('|')
	if len(bits) > 1:
		for char in bits[1]:
			chars.add(char)

print(''.join(chars))


#!/usr/bin/env python
# File: load-resnet.py
# Author: Eric Yujia Huang yujiah1@andrew.cmu.edu
#         Yuxin Wu <ppwwyyxx@gmail.com>

import cv2
import functools
import tensorflow as tf
import argparse
import os
import re
import numpy as np
import six
from six.moves import zip
from tensorflow.contrib.layers import variance_scaling_initializer

from tensorpack import *
from tensorpack.utils import logger
from tensorpack.utils.stats import RatioCounter
from tensorpack.tfutils.summary import *
from tensorpack.dataflow.dataset import ILSVRCMeta, ILSVRC12

from imagenet_utils import eval_on_ILSVRC12, get_imagenet_dataflow, ImageNetModel
from resnet_model import resnet_group, resnet_bottleneck

DEPTH = None
CFG = {
    50: ([3, 4, 6, 3]),
    101: ([3, 4, 23, 3]),
    152: ([3, 8, 36, 3])
}

class Model(ModelDesc):
    def __init__(self, target_shape):
        super(Model, self).__init__()
        self.target_shape = target_shape

    def _get_inputs(self):
        return [InputDesc(tf.float32, [None, self.target_shape,
                                       self.target_shape, 3], 'input'),
                InputDesc(tf.int32, [None], 'label')]

    def _build_graph(self, inputs):
        image, label = inputs
        blocks = CFG[DEPTH]

        bottleneck = functools.partial(resnet_bottleneck, stride_first=True)

        # tensorflow with padding=SAME will by default pad [2,3] here.
        # but caffe conv with stride will pad [3,3]
        image = tf.pad(image, [[0, 0], [3, 3], [3, 3], [0, 0]])
        image = tf.transpose(image, [0, 3, 1, 2])
        with argscope([Conv2D, MaxPooling, GlobalAvgPooling, BatchNorm],
                      data_format='NCHW'), \
                argscope(Conv2D, nl=tf.identity, use_bias=False,
                         W_init=variance_scaling_initializer(mode='FAN_OUT')):
            logits = (LinearWrap(image)
                      .Conv2D('conv0', 64, 7, stride=2, nl=BNReLU, padding='VALID')
                      .MaxPooling('pool0', shape=3, stride=2, padding='SAME')
                      .apply(resnet_group, 'group0', bottleneck, 64, blocks[0], 1)
                      .apply(resnet_group, 'group1', bottleneck, 128, blocks[1], 2)
                      .apply(resnet_group, 'group2', bottleneck, 256, blocks[2], 2)
                      .apply(resnet_group, 'group3', bottleneck, 512, blocks[3], 2)
                      .GlobalAvgPooling('gap')
                      .FullyConnected('linear', 1000, nl=tf.identity)())
        prob = tf.nn.softmax(logits, name='prob')
        ImageNetModel.compute_loss_and_error(logits, label)

def get_inference_augmentor(target_shape):
    # load ResNet mean from Kaiming:
    # from tensorpack.utils.loadcaffe import get_caffe_pb
    # obj = get_caffe_pb().BlobProto()
    # obj.ParseFromString(open('ResNet_mean.binaryproto').read())
    # pp_mean_224 = np.array(obj.data).reshape(3, 224, 224).transpose(1,2,0)

    meta = ILSVRCMeta()
    pp_mean = meta.get_per_pixel_mean()

    pp_mean_crop = pp_mean[16:-16, 16:-16, :]

    transformers = [
        imgaug.ResizeShortestEdge(256),
        imgaug.CenterCrop((224, 224)),
        imgaug.MapImage(lambda x: x - pp_mean_crop),
    ]

    if target_shape != 224:
        transformers.append(imgaug.ResizeShortestEdge(target_shape, cv2.INTER_CUBIC))

    return transformers

def run_test(params, input, target_shape):
    pred_config = PredictConfig(
        model=Model(target_shape),
        session_init=DictRestore(params),
        input_names=['input'],
        output_names=['prob']
    )
    predict_func = OfflinePredictor(pred_config)

    prepro = imgaug.AugmentorList(get_inference_augmentor(target_shape))
    im = cv2.imread(input).astype('float32')
    im = prepro.augment(im)
    im = np.reshape(im, (1, target_shape, target_shape, 3))
    outputs = predict_func(im)
    prob = outputs[0]

    ret = prob[0].argsort()[-10:][::-1]
    print(ret)
    meta = ILSVRCMeta().get_synset_words_1000()
    print([meta[k] for k in ret])

def name_conversion(caffe_layer_name):
    """ Convert a caffe parameter name to a tensorflow parameter name as
        defined in the above model """
    # beginning & end mapping
    NAME_MAP = {'bn_conv1/beta': 'conv0/bn/beta',
                'bn_conv1/gamma': 'conv0/bn/gamma',
                'bn_conv1/mean/EMA': 'conv0/bn/mean/EMA',
                'bn_conv1/variance/EMA': 'conv0/bn/variance/EMA',
                'conv1/W': 'conv0/W', 'conv1/b': 'conv0/b',
                'fc1000/W': 'linear/W', 'fc1000/b': 'linear/b'}
    if caffe_layer_name in NAME_MAP:
        return NAME_MAP[caffe_layer_name]

    s = re.search('([a-z]+)([0-9]+)([a-z]+)_', caffe_layer_name)
    if s is None:
        s = re.search('([a-z]+)([0-9]+)([a-z]+)([0-9]+)_', caffe_layer_name)
        layer_block_part1 = s.group(3)
        layer_block_part2 = s.group(4)
        assert layer_block_part1 in ['a', 'b']
        layer_block = 0 if layer_block_part1 == 'a' else int(layer_block_part2)
    else:
        layer_block = ord(s.group(3)) - ord('a')
    layer_type = s.group(1)
    layer_group = s.group(2)

    layer_branch = int(re.search('_branch([0-9])', caffe_layer_name).group(1))
    assert layer_branch in [1, 2]
    if layer_branch == 2:
        layer_id = re.search('_branch[0-9]([a-z])/', caffe_layer_name).group(1)
        layer_id = ord(layer_id) - ord('a') + 1

    TYPE_DICT = {'res': 'conv{}', 'bn': 'conv{}/bn'}
    layer_type = TYPE_DICT[layer_type].format(layer_id if layer_branch == 2 else 'shortcut')

    tf_name = caffe_layer_name[caffe_layer_name.index('/'):]
    tf_name = 'group{}/block{}/{}'.format(
        int(layer_group) - 2, layer_block, layer_type) + tf_name
    return tf_name

def convert_param_name(param):
    resnet_param = {}
    for k, v in six.iteritems(param):
        try:
            newname = name_conversion(k)
        except Exception:
            logger.error("Exception when processing caffe layer {}".format(k))
            raise
        logger.info("Name Transform: " + k + ' --> ' + newname)
        resnet_param[newname] = v
    return resnet_param

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--load', required=True,
                        help='.npy model file generated by tensorpack.utils.loadcaffe')
    parser.add_argument('-d', '--depth', help='resnet depth', required=True, type=int, choices=[50, 101, 152])
    parser.add_argument('--input', help='an input image')
    parser.add_argument('--convert', help='npz output file to save the converted model')
    parser.add_argument('--eval', help='ILSVRC dir to run validation on')
    parser.add_argument('--image-size', choices=[224, 75],
                        help='image size to feed to resnet')
    parser.add_argument('--dataset', choices=['val', 'train'])

    args = parser.parse_args()
    DEPTH = args.depth

    param = np.load(args.load, encoding='latin1').item()
    param = convert_param_name(param)
    target_shape = args.image_size

    if args.convert:
        assert args.convert.endswith('.npz')
        np.savez_compressed(args.convert, **param)

    if args.eval:
        ds = get_imagenet_dataflow(args.eval, args.dataset, 128,
                                   get_inference_augmentor(target_shape))
        eval_on_ILSVRC12(Model(target_shape), DictRestore(param), ds)
    elif args.input:
        run_test(param, args.input, target_shape)


import os
import json

import cv2
import numpy as np
from torch.utils.data import Dataset

# def detection_collate(batch):
#     return tuple(zip(*batch))

# def detection_collate_list(batch):
#     return tuple(map(list, zip(*batch)))

def detection_collate(batch):
    return tuple(map(list, zip(*batch)))

class DetectionDataset(Dataset):
    def __init__(self, data_path, json_path=None, transforms=None):
        """
        Для тренировки необходимо подавать json_file
        Для inference не надо подавать json_file
        """

        super(DetectionDataset, self).__init__()

        # Сохраняем входные параметры
        self.data_path = data_path
        self.json_path = json_path
        self.transforms = transforms

        if self.json_path is None:
            # Читаем все файлы в директории
            self.images = os.listdir(self.data_path)
            # Фильтруем маски и вырезанные номеры
            self.images = filter(lambda x: ('.mask.' not in x) and ('.box' not in x), self.images)
            # Сортируем файлы
            self.images = sorted(self.images, key=lambda x: int(os.path.splitext(x)[0]))
            # Добавляем папку
            self.images = map(lambda x: os.path.join(self.data_path, x), self.images)
            # Переводим в список
            self.images = list(self.images)
            # Переводим в массив списков с ключем file
            raw_info = [{'file': x} for x in self.images]
        else:
            basepath = os.path.dirname(self.json_path)
            # Читаем json файл
            with open(json_path, 'r') as in_file:
                raw_info = json.load(in_file)
                for i in range(len(raw_info)):
                    raw_info[i]['file'] = os.path.join(basepath, raw_info[i]['file'])

        # Отфильтруем плохие изображения (нулевой вес)
        self.info = []
        for info in raw_info:
            if os.path.getsize(info['file']) != 0:
                self.info.append(info)

    def __len__(self):
        return len(self.info)

    def get_height_and_width(self, item):
        image = cv2.imread(self.info[item]['file']).astype(np.float32)
        height, width, _ = image.shape
        return height, width

    def __getitem__(self, item):
        image = cv2.imread(self.info[item]['file']).astype(np.float32)
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) / 255.

        target = None

        if 'nums' in self.info[item]:
            boxes = []
            labels = []
            image_id = [item]
            area = []
            masks = []

            # Проходимся по всем bbox'ам
            for box in self.info[item]['nums']:
                bbox = np.array(box['box'])

                # Находим крайние точки
                x0, y0 = np.min(bbox[:, 0]), np.min(bbox[:, 1])
                x1, y1 = np.max(bbox[:, 0]), np.max(bbox[:, 1])
                boxes.append([x0, y0, x1, y1])

                # Формируем label
                labels.append(1)

                # Формируем area
                area.append((x1 - x0) * (y1 - y0))

                # Формируем mask
                mask = np.zeros(shape=image.shape[:-1], dtype=np.uint8)
                cv2.fillConvexPoly(mask, bbox, (255, 255, 255))
                mask = mask / 255.
                masks.append(mask)

            target = {
                'boxes': np.array(boxes).astype(np.float32),
                'labels': np.array(labels).astype(np.int64),
                'image_id': np.array(image_id).astype(np.int64),
                'area': np.array(area).astype(np.float32),
                'masks': np.array(masks).astype(np.uint8)
            }

        if self.transforms is not None:
            image, target = self.transforms(image, target)

        return image, target


import copy
from glob import glob
import cv2
from matplotlib import scale
import numpy as np
import torch
from pytorch3d import transforms
import scipy.ndimage as img_fn

import pickle
from pathlib import Path
import smplx
import tqdm
import open3d as o3d

# 30 fps. We want to have input 5 seconds and predict 10 seconds. So maybe sample 3 fps, 15 input frames and 30 prediction frames.
def get_fps(things, offset=5, samples=50):
    """things is a list of dictionaries describing files with a 'tstamp' key of datetime.time value type.
    Sample evenly adjacent tstamp differences to get fps.
    Seems about 30. Ofc could have not done i+1 - i but ahhhhh."""
    import datetime
    samples = np.arange(offset, len(things)-2, len(things)//samples)
    deltas = [(things[i]['tstamp'], things[i+1]['tstamp']) for i in samples]
    deltas = [datetime.datetime.combine(datetime.date.today(), b) - datetime.datetime.combine(datetime.date.today(), a) for a, b in deltas]
    return list(map(lambda x: x.total_seconds(), deltas))

def normalize_euler_angles(base_rot_matrix, euler_angles):
    old_rot_matrix = transforms.euler_angles_to_matrix(euler_angles, convention='XYZ')
    new_rot_matrix = old_rot_matrix @ base_rot_matrix.transpose(1, 2)
    new_euler_angles = transforms.matrix_to_euler_angles(new_rot_matrix, convention='XYZ')
    return new_euler_angles

def get_joint_locations(data_dict, body_model):
    betas, body_pose, global_orient, transl = extract_data(data_dict)
    out = body_model(return_joints=True, betas=betas, body_pose=body_pose, global_orient=global_orient, transl=transl)
    return out.joints[:, :21].squeeze().detach().cpu().numpy(), (out.vertices.detach().cpu().numpy().squeeze())

# TODO body_model is used here without definition - pass around with global or as param?
def normalized_joint_locations1(data_dict, base_rot_matrix=torch.eye(3).unsqueeze(0), base_transl=torch.zeros(3), body_model=None):
    """normalises wrt rotation and translation. body_model shouldn't be None I'm just lazy."""
    betas, body_pose, euler_angles, transl = extract_data(data_dict)
    transl = transl - base_transl
    euler_angles = normalize_euler_angles(base_rot_matrix, euler_angles)

    out = body_model(return_joints=True, betas=betas, body_pose=body_pose, global_orient=euler_angles, transl=transl)
    joint_locs = out.joints[:, :25]  # 1, 25, 3
    return joint_locs

def normalized_joint_locations(in_data_dicts, pred_data_dicts, body_model):
    # TODO rotation normalisation only makes sense about the vector which aligns with gravity, or maybe not at all?
    #  for now we take it out
    _betas, _body_pose, base_euler_angles, base_transl = extract_data(in_data_dicts[-1])
    base_rot_matrix = transforms.euler_angles_to_matrix(base_euler_angles, convention='XYZ')
    base_rot_matrix=torch.eye(3).reshape((1, 3, 3))

    in_joint_locations = torch.cat(
        [normalized_joint_locations1(data_dict, base_rot_matrix, base_transl, body_model) for data_dict in in_data_dicts], dim=0)
    pred_joint_locations = torch.cat(
        [normalized_joint_locations1(data_dict, base_rot_matrix, base_transl, body_model) for data_dict in pred_data_dicts], dim=0)

    return in_joint_locations, pred_joint_locations

def normalized_joint_locations_world(in_data_dict: dict, body_model, cam2world: torch.tensor):
    """Converts joint locations of our smplx data dict into world coordinate system.
    In a sequence you'd want to further translation normalise wrt to the first skeleton pelvis position.
    Then to reverse you'd reverse translate and then inverse world2cam.
    """
    joint_locs = data_dict_to_joint_locs(in_data_dict, body_model)  # 1, 25, 3

    cam_R = cam2world[:3, :3].reshape([3, 3])
    cam_t = cam2world[:3, 3].reshape([1, 3])
    joint_locs = torch.matmul(cam_R, joint_locs.permute(0, 2, 1)).permute(0, 2, 1) + cam_t

    return joint_locs

def data_dict_to_joint_locs(in_data_dict, body_model):
    betas, body_pose, global_orient, transl = extract_data(in_data_dict)
    out = body_model(return_joints=True, betas=betas, body_pose=body_pose, global_orient=global_orient, transl=transl)
    joint_locs = out.joints[:, :25]  # 1, 25, 3
    return joint_locs

def cam2world_conv(joint_locs: torch.Tensor, cam2world: torch.Tensor):
    """joint_locs: ? x n x 3 or n x 3 I think
    cam2world: 4x4 R and t matrix"""
    if len(joint_locs.shape) == 2:
        joint_locs = joint_locs.unsqueeze(0)
    if len(joint_locs.shape) != 3:
        raise Exception('shape should be length 2 or 3')

    cam_R = cam2world[:3, :3].reshape([3, 3])
    cam_t = cam2world[:3, 3].reshape([1, 3])
    joint_locs = torch.matmul(cam_R, joint_locs.permute(0, 2, 1)).permute(0, 2, 1) + cam_t
    return joint_locs

def world2cam_conv(joint_locs: torch.Tensor, cam2world: torch.tensor):
    """joint_locs: ? x n x 3 or n x 3 I think
    cam2world: 4x4 R and t matrix"""
    if len(joint_locs.shape) == 2:
        joint_locs = joint_locs.unsqueeze(0)
    if len(joint_locs.shape) != 3:
        raise Exception('shape should be length 2 or 3')

    cam_R = cam2world[:3, :3].reshape([3, 3])
    cam_t = cam2world[:3, 3].reshape([1, 3])
    joint_locs = torch.matmul(torch.inverse(cam_R), (joint_locs - cam_t).permute(0, 2, 1)).permute(0, 2, 1)
    return joint_locs

def extract_data(data_dict):
    betas = torch.Tensor(data_dict['betas'])
    body_pose = torch.Tensor(data_dict['body_pose'])
    global_orient = torch.Tensor(data_dict['global_orient'])
    transl = torch.Tensor(data_dict['transl'])
    return betas, body_pose, global_orient, transl

def l1_inpainting(f, mask, theta=0.001, maxIter=5000):
    u = np.array((1 - mask) * f)
    mask = np.pad(mask, pad_width=1, mode='edge')
    f = np.pad(f, pad_width=1, mode='edge')

    for i in range(maxIter):
        u_b = np.pad(u, pad_width=1, mode='edge')
        ux = img_fn.sobel(u, axis=1)
        uy = img_fn.sobel(u, axis=0)
        phi_prime = np.pad( 1 /np.sqrt(1 + (ux**2 +uy**2 ) /theta**2), pad_width=1, mode='edge')

        u_bp = np.roll(u_b, 1, axis=1)
        u_bn = np.roll(u_b, -1, axis=1)
        u_bu = np.roll(u_b, 1, axis=0)
        u_bd = np.roll(u_b, -1, axis=0)

        a_p = 0.5 *(phi_prime + np.roll(phi_prime, 1, axis=1))
        a_n = 0.5 *(phi_prime + np.roll(phi_prime, -1, axis=1))
        a_u = 0.5 *(phi_prime + np.roll(phi_prime, 1, axis=0))
        a_d = 0.5 *(phi_prime + np.roll(phi_prime, -1, axis=0))
        a_c = (a_p + a_n + a_u + a_d)

        u_c = ((1 - mask) * f + 1.0 * mask * (a_p * u_bp + a_n * u_bn + a_u * u_bu + a_d * u_bd)
               ) / (1 - mask + 1.0 * mask * a_c)
        u = np.copy(u_c[1:-1, 1:-1])

    return u

def get_smplx_body_model(smplx_model_path):
    body_model = smplx.create(smplx_model_path,
                            model_type='smplx',  ## smpl, smpl+h, or smplx?
                            gender='neutral', ext='npz',  ## file format
                            num_pca_comps=12,  ## MANO hand pose pca component
                            create_global_orient=True,
                            create_body_pose=True,
                            create_betas=True,
                            create_left_hand_pose=True,
                            create_right_hand_pose=True,
                            create_expression=True,
                            create_jaw_pose=True,
                            create_leye_pose=True,
                            create_reye_pose=True,
                            create_transl=True,
                            batch_size=1  ## how many bodies in a batch?
                            )
    body_model.eval()
    return body_model

# TODO what should this be?
BACKGROUND_PROX = 10.0

DEPTH_SCALE = 8 * 1000.

LIMBS = [
(0, 1),  # head_center -> neck
(1, 2),  # neck -> right_clavicle
(2, 3),  # right_clavicle -> right_shoulder
(3, 4),  # right_shoulder -> right_elbow
(4, 5),  # right_elbow -> right_wrist
(1, 6),  # neck -> left_clavicle
(6, 7),  # left_clavicle -> left_shoulder
(7, 8),  # left_shoulder -> left_elbow
(8, 9),  # left_elbow -> left_wrist
(1, 10),  # neck -> spine0
(10, 11),  # spine0 -> spine1
(11, 12),  # spine1 -> spine2
(12, 13),  # spine2 -> spine3
(13, 14),  # spine3 -> spine4
(14, 15),  # spine4 -> right_hip
(15, 16),  # right_hip -> right_knee
(16, 17),  # right_knee -> right_ankle
(14, 18),  # spine4 -> left_hip
(18, 19),  # left_hip -> left_knee
(19, 20),  # left_knee -> left_ankle
]

def scale_camera_dict(camera_dict, scale=DEPTH_SCALE):
    camera_dict['f'][0] *= scale
    camera_dict['f'][1] *= scale

    camera_dict['camera_mtx'][0][0] *= scale
    camera_dict['camera_mtx'][1][1] *= scale

def proximity_map(depth_map, skeleton_dict, body_model, depth_cam, alignment_cam):
    joint_locations, body_points =  get_joint_locations(skeleton_dict, body_model)

    skeleton = o3d.geometry.PointCloud()
    skeleton.points = o3d.utility.Vector3dVector(body_points)

    depth_cam = copy.deepcopy(depth_cam)
    scale_camera_dict(depth_cam)

    o3d_depth_map = o3d.geometry.Image(depth_map.astype(np.float32))

    h, w = depth_map.shape

    # TODO figure out if depth_scale, depth_trunc necessary
    pcd = o3d.geometry.PointCloud.create_from_depth_image(
    o3d_depth_map,
    o3d.camera.PinholeCameraIntrinsic(w, h, depth_cam['f'][0], depth_cam['f'][1], depth_cam['c'][0], depth_cam['c'][1]),
    # depth_scale=1000.,

    )  # cam coordinate

    points = np.asarray(pcd.points)
    points[:, 2] /= DEPTH_SCALE
    pcd.points = o3d.utility.Vector3dVector(points)

    points_color_aligned = align_point_clouds(points, depth_cam, alignment_cam)
    pcd_color_aligned = o3d.geometry.PointCloud()
    pcd_color_aligned.points = o3d.utility.Vector3dVector(points_color_aligned)

    body_bps = point_proximity_map(points_color_aligned, joint_locations)

    # TODO human mask
    depth_mask_ind = np.where(depth_map.flatten() == 0.)[0]
    # where we have points in the point cloud
    depth_nomask_ind = np.asarray(list(set(range(h*w))-set(depth_mask_ind)))  # indices of nonmasked pixels
    # TODO set sky mask where depth_aligned > MAX_DEPTH.
    # depth_mask_sky_ind = ((depth_aligned.flatten() == 0) * (human_mask==False)).reshape(h*w)  # indices of sky

    body_bps_full = np.zeros([h * w])
    body_bps_full[depth_nomask_ind] = body_bps
    body_bps_full[depth_mask_ind] = BACKGROUND_PROX

    body_bps_full = body_bps_full.reshape((h, w))  # body bps feature map, [h, w]
    body_bps_full = body_bps_full.astype(np.float32)

    return pcd, pcd_color_aligned, skeleton, body_bps_full

def align_point_clouds(points, depth_cam, alignment_cam):
    R1 = np.array(depth_cam['R'])
    # equivalent seems
    # R1 = np.array(transforms.euler_angles_to_matrix(torch.Tensor(R), convention='XYZ'))
    R1, _ = cv2.Rodrigues(R1)
    # same
    T1 = np.array(depth_cam['T'])

    R2 = np.array(alignment_cam['R'])
    # equivalent seems
    # R1 = np.array(transforms.euler_angles_to_matrix(torch.Tensor(R), convention='XYZ'))
    R2, _ = cv2.Rodrigues(R2)
    # same
    T2 = np.array(alignment_cam['T'])

    points_color_aligned = (R2 @ (R1.T @ (points - T1).T)).T + T2
    return points_color_aligned

def point_proximity_map(points, joint_locations):

    A = joint_locations[np.asarray(LIMBS)[:, 0]]  # [n_limb, 3]
    B = joint_locations[np.asarray(LIMBS)[:, 1]]  # [n_limb, 3]
    n_pt = points.shape[0]
    n_limb = A.shape[0]

    A = np.tile(A, (n_pt, 1)).reshape(n_pt*n_limb, 3)  # [n_pt, n_limb, 3], n_pt=n_bps=scene_verts.shape[0]
    B = np.tile(B, (n_pt, 1)).reshape(n_pt*n_limb, 3)
    P = np.tile(points, n_limb).reshape(n_pt*n_limb, 3)

    AB = B - A
    AP = P - A
    BP = P - B

    temp_1 = np.multiply(AB, AP).sum(axis=-1)  # [n_pt, n_limb]
    temp_2 = np.multiply(-AB, BP).sum(axis=-1)  # [n_pt, n_limb]
    mask_1 = np.where(temp_1 <= 0)[0]   # angle between AB and AP >= 90
    mask_2 = np.where((temp_1 > 0) * (temp_2 <= 0))[0]  # angle between AB and AP < 90 and angle between BA and BP >= 90
    mask_3 = np.where((temp_1 > 0) * (temp_2 > 0))[0]   # angle between AB and AP < 90 and angle between BA and BP < 90
    if len(mask_1) + len(mask_2) + len(mask_3) != n_pt*n_limb:
        print('[distance calculation] num of verts does not match!')

    dist_1 = np.sqrt(np.sum((P[mask_1]-A[mask_1])**2, axis=-1))  # [n_mask_1]
    dist_2 = np.sqrt(np.sum((P[mask_2]-B[mask_2])**2, axis=-1))  # [n_mask_2]

    x = np.multiply(AB[mask_3], AP[mask_3]).sum(axis=-1) / np.multiply(AB[mask_3], AB[mask_3]).sum(axis=-1)  # [n_mask_3]
    x = x.repeat(3).reshape(-1,3)
    C = x * AB[mask_3] + A[mask_3]  # C: [n_mask_3, 3], the projected point of P on line segment AB
    dist_3 = np.sqrt(np.sum((P[mask_3]-C)**2, axis=-1))  # n_mask_3

    dist = np.zeros(n_pt*n_limb)
    dist[mask_1] = dist_1
    dist[mask_2] = dist_2
    dist[mask_3] = dist_3
    dist = dist.reshape(n_pt, n_limb)  # [n_pt, n_limb], distance from each point in scene verts to each limb
    body_bps = np.min(dist, axis=-1)   # [n_pt]

    return body_bps


import astar
import cv2
from queue import PriorityQueue
import numpy as np
from threading import Timer

NODE_Obstacle = "obstacle"
NODE_Free = "free"
NODE_InOpenList = 'open'
NODE_InCloseList = 'close'
NODE_Unknown = 4

adjacent=[]
adjacent.extend([(1,0),(1,1),(1,-1),(0,1),(0,-1),(-1,0),(-1,1),(-1,-1)])
print(adjacent)
Inf = float('inf')
class Node(object):
    def __init__(self,x=0,y=0,g=Inf,f=Inf,next_=None):
        self.x = x
        self.y = y
        self.h = 0
        self.g = g
        self.f = f
        self.status = NODE_Free
        self.ParentNode=next_
    def point(self):
        return (self.x,self.y)

    def __lt__(self,other):
        if self.f < other.f:
            return True
        else:
            return False
    def __repr__(self):
        return str((self.x,self.y,self.f,self.status))

class mAstar(object):
    def __init__(self):
        self.path=[]
        self.panel = "Img"
        self.img=None
        self.graph = {}
        self.start=None
        self.end=None

    def refresh(self):
        self.show()
        self.mTimer = Timer(20,self.refresh)
        self.mTimer.start()

    def load_img(self,imgpath):
        self.img = cv2.imread(imgpath,cv2.IMREAD_GRAYSCALE)
        self.img_raw = self.img.copy()

    def mousecallback(self,event,x,y,flags,param):
        if event == cv2.EVENT_LBUTTONUP:
            if self.start!=None:
                self.end = Node(x,y)
                cv2.circle(self.img,(x,y),1,0,1)
                cv2.imshow(self.panel,self.img)
                self.search(self.start,self.end)
                self.start=None
                return
            self.img = self.img_raw.copy()
            self.start = Node(x,y)
            cv2.circle(self.img,(x,y),1,0,1)
            cv2.imshow(self.panel,self.img)

    def show(self):
        cv2.namedWindow(self.panel,cv2.WINDOW_NORMAL)
        cv2.setMouseCallback(self.panel,self.mousecallback)
        cv2.imshow(self.panel,self.img)
        cv2.waitKey()

    def search(self,start,end):
        self.img[start.y][start.x]=50
        self.img[end.y][end.x]=50
        openlist = PriorityQueue()
        closelist= []
        width = self.img.shape[1]
        height=self.img.shape[0]

        # input()
        t = cv2.getTickCount()
        print(t)
        for x in range(0,width):
            for y in range(0,height):
                tnode = Node(x,y)
                if self.img[y,x]>250:
                    tnode.status = NODE_Free
                else:
                    tnode.status= NODE_Obstacle
                    # print('obstacle',tnode)
                if x not in self.graph.keys():
                    self.graph[x]={}
                self.graph[x][y]=tnode
        print(cv2.getTickCount()-t)
        tnode = Node(start.x,start.y,0,0)
        tnode.status = NODE_Free
        self.graph[start.x][start.y]=tnode
        tnode = Node(end.x,end.y)
        tnode.status = NODE_Free
        self.graph[end.x][end.y]=tnode

        openlist.put(self.graph[start.x][start.y])

        t=cv2.getTickCount()
        print('start:',t)
        while not openlist.empty():
            # input()
            current = openlist.get()
            # print('\nclose:',current)
            if current.x == end.x and current.y == end.y:
                end=current
                print("Found.\n")

                break
            current.status = NODE_InCloseList
            self.img[current.y,current.x]=128
            self.img[start.y][start.x]=50
            for i in range(0,8):
                # input()
                xnow = current.x + adjacent[i][0]
                ynow = current.y + adjacent[i][1]
                if 0<=xnow<width and 0<=ynow<height:
                    neighbor = self.graph[xnow][ynow]
                    if neighbor.status == NODE_Obstacle or neighbor.status == NODE_InCloseList:
                        # print('ignore:',neighbor)
                        continue
                    cost = self.movementcost(current,neighbor) + current.g
                    if cost >= neighbor.g:
                        # print('cost {0} bigger than {1}:'.format(cost,neighbor.g))
                        continue
                    neighbor.ParentNode = current
                    neighbor.g = cost
                    neighbor.f = neighbor.g+self.heuristic(neighbor,end)
                    # print("put:",neighbor)
                    openlist.put(neighbor)
                    # cv2.namedWindow(self.panel,cv2.WINDOW_NORMAL)
            # cv2.imshow(self.panel,self.img)

        print('finished')
        node=end.ParentNode
        print(node)
        print(cv2.getTickCount()-t)
        while node.x!=start.x or node.y!=start.y:
            cv2.circle(self.img,node.point(),1,0,1)
            self.img[node.y][node.x]=80
            node = node.ParentNode
            print(node)
            cv2.imshow(self.panel,self.img)
            cv2.waitKey(1)
        cv2.waitKey(2)

    def movementcost(self,A,B):
        stepX = abs(A.x - B.x)
        stepY = abs(A.y - B.y)
        step = stepX + stepY
        if step == 1:
            cost = 10
        else:
            cost = 14
        return cost

    def heuristic(self,A,B):
        h = 10*((abs(A.x-B.x)+abs(A.y-B.y)))
        return h

if __name__ == "__main__":

    pf=mAstar()
    pf.load_img("test2.jpg")
    pf.show()


from pyexpat import model
from tabnanny import verbose
from django.db import models

# Create your models here.
class Post( models.Model ):
    titulo = models.CharField(
        max_length=194,
        verbose_name="Título"
    )

    conteudo = models.TextField(
        verbose_name="Conteúdo"
    )

    categoria = models.CharField(
        choices=[
            ('django', 'django'),
            ('python', 'python')
            ],
        default='django',
        verbose_name="Categoria",
        max_length=6
    )

    slug = models.CharField(
        verbose_name="Slug",
        max_length=194,
        blank=False,
    )

    data_publicado = models.DateTimeField(
        auto_now_add=True,
        verbose_name="Data de publicação"
    )

    class Meta:
        verbose_name = "Post"
        verbose_name_plural = "Posts"
        db_table = "post"

    def __str__(self):
        return self.titulo