import pandas as pd

def largest_orders(orders: pd.DataFrame) -> pd.DataFrame:
    orders = orders.groupby('customer_number')['order_number'].count().reset_index()
    orders.sort_values(by='order_number',ascending=False,inplace=True)
    return orders.iloc[:1,[0]]

data = [[1, 1], [2, 2], [3, 3], [4, 3]]
orders = pd.DataFrame(data, columns=['order_number', 'customer_number']).astype(
    {'order_number':'Int64', 'customer_number':'Int64'})

print(largest_orders(orders))

Data Agregation/Customer Placing the Largest Number of Orders.py


n,m = tuple(map(int,input().split()))
grid = [
    list(map(int,input().split()))
    for _ in range(n)
]

# 주어진 k에 대하여 마름모의 넓이를 반환합니다.
def get_area(k):
    return k*k+(k+1)*(k+1)

# 주어진 k에 대하여 채굴 가능한 금의 개수를 반환합니다.
def get_num_of_gold(row,col,k):
    return sum([
        grid[i][j]
        for i in range(n)
        for j in range(n)
        if abs(row-i) + abs(col-j) <= k
    ])

max_gold=0

# 격자의 각 위치가 마름모의 중앙일 때 채굴 가능한 금의 개수를 구합니다.
for row in range(n):
    for col in range(n):
        for k in range(2*(n-1)+1):
            num_of_gold = get_num_of_gold(row,col,k)

            # 손해를 보지 않으면서 채굴할 수 있는 최대 금의 개수를 저장합니다.
            if num_of_gold * m >= get_area(k):
                max_gold = max(max_gold, num_of_gold)

print(max_gold)

김정수/시뮬레이션/1주차/ch1/금 채굴하기.py


from tkinter import *

import numpy as np
from PIL import ImageTk, Image
from tkinter import filedialog
import cv2

imgX = Image.open("Q2.png")
x = None

# open image function
def open_img():
    # Select the ImageName from a folder
    global x
    x = openfilename()
    # opens the image
    img = Image.open(x)
    # resize the image and apply a high-quality down sampling filter
    img = img.resize((640, 480), Image.ANTIALIAS)
    # PhotoImage class is used to add image to widgets, icons etc
    img = ImageTk.PhotoImage(img)
    # create a label
    panel = Label(root, image=img)
    # set the image as img
    panel.image = img
    panel.grid(row=2, column=1)

# function to get file name
def openfilename():
    # open file dialog box to select image
    # The dialogue box has a title "Open"
    filename = filedialog.askopenfilename(title='Open')
    return filename

# transform to grayscale using PIL
def gray():
    global imgX
    imgX = Image.open(x)
    imgX = imgX.resize((640, 480), Image.ANTIALIAS).convert('L')
    imgX1 = ImageTk.PhotoImage(imgX)
    panel = Label(root, image=imgX1)
    panel.image = imgX1
    panel.grid(row=2, column=2)
    return

# invert image using cv2
def invert():
    global imgX
    # read & process image using cv2
    imgX1 = cv2.imread(x)
    imgX1 = cv2.bitwise_not(imgX1)

    # convert cv2 image to imageTk
    b, g, r = cv2.split(imgX1)
    imgX1 = cv2.merge((r, g, b))
    print(imgX1)
    imgX = Image.fromarray(imgX1)

    imgX = imgX.resize((640, 480), Image.ANTIALIAS)
    imgX1 = ImageTk.PhotoImage(image=imgX)
    panel = Label(root, image=imgX1)
    panel.image = imgX1
    panel.grid(row=2, column=2)

def grayslice():
    global imgX
    # read & process image using cv2
    imgX1 = cv2.imread(x, 0)
    # Find width and height of image
    row, column = imgX1.shape

    # Specify the min and max range
    min_range = 10
    max_range = 100

    # Loop over the input image and if pixel value lies in desired range set it to 255 otherwise set it to 0.
    for i in range(row):
        for j in range(column):
            if min_range < imgX1[i, j] < max_range:
                imgX1[i, j] = 255
            else:
                imgX1[i, j] = 0

    # convert cv2 image to imageTk
    imgX = Image.fromarray(imgX1)

    imgX = imgX.resize((640, 480))
    imgX1 = ImageTk.PhotoImage(image=imgX)
    panel = Label(root, image=imgX1)
    panel.image = imgX1
    panel.grid(row=2, column=2)
    return

# Bit plane slicing
def bitPlaneSlice():
    global imgX
    imgX = cv2.imread(x, 0)
    # Iterate over each pixel and change pixel value to binary using np.binary_repr() and store it in a list.
    lst = []
    for i in range(imgX.shape[0]):
        for j in range(imgX.shape[1]):
            lst.append(np.binary_repr(imgX[i][j], width=8))  # width = no. of bits

    # We have a list of strings where each string represents binary pixel value. To extract bit planes we need to
    # iterate over the strings and store the characters corresponding to bit planes into lists. Multiply with 2^(n-1)
    # and reshape to reconstruct the bit image.
    eight_bit_img = (np.array([int(i[0]) for i in lst], dtype=np.uint8) * 128).reshape(imgX.shape[0], imgX.shape[1])
    seven_bit_img = (np.array([int(i[1]) for i in lst], dtype=np.uint8) * 64).reshape(imgX.shape[0], imgX.shape[1])
    six_bit_img = (np.array([int(i[2]) for i in lst], dtype=np.uint8) * 32).reshape(imgX.shape[0], imgX.shape[1])
    five_bit_img = (np.array([int(i[3]) for i in lst], dtype=np.uint8) * 16).reshape(imgX.shape[0], imgX.shape[1])
    four_bit_img = (np.array([int(i[4]) for i in lst], dtype=np.uint8) * 8).reshape(imgX.shape[0], imgX.shape[1])
    three_bit_img = (np.array([int(i[5]) for i in lst], dtype=np.uint8) * 4).reshape(imgX.shape[0], imgX.shape[1])
    two_bit_img = (np.array([int(i[6]) for i in lst], dtype=np.uint8) * 2).reshape(imgX.shape[0], imgX.shape[1])
    one_bit_img = (np.array([int(i[7]) for i in lst], dtype=np.uint8) * 1).reshape(imgX.shape[0], imgX.shape[1])

    # Concatenate these images for ease of display using cv2.hconcat()
    finalR = cv2.hconcat([eight_bit_img, seven_bit_img, six_bit_img, five_bit_img])
    finalV = cv2.hconcat([four_bit_img, three_bit_img, two_bit_img, one_bit_img])

    # Vertically concatenate
    final = cv2.vconcat([finalR, finalV])

    # display edited image.
    imgX = Image.fromarray(final)
    imgX = imgX.resize((640, 480), Image.ANTIALIAS)
    imgX1 = ImageTk.PhotoImage(image=imgX)
    panel = Label(root, image=imgX1)
    panel.image = imgX1
    panel.grid(row=2, column=2)
    return

# Contrast Stretching
def contrastStretch():
    global imgX
    # Open the image.
    imgX1 = cv2.imread(x, 0)
    # Define parameters.
    r1 = 100
    s1 = 0
    r2 = 200
    s2 = 255
    row, column = imgX1.shape
    for i in range(row):
        for j in range(column):
            if 0 <= imgX1[i, j] <= r1:
                imgX1[i, j] = (s1 / r1) * imgX1[i, j]
            elif r1 < imgX1[i, j] <= r2:
                imgX1[i, j] = ((s2 - s1) * (imgX1[i, j] - r1) / (r2 - r1)) + s1
            else:
                imgX1[i, j] = ((255 - s2) * (imgX1[i, j] - r2) / (255 - r2)) + s2

    # display edited image.
    imgX = Image.fromarray(imgX1)
    imgX = imgX.resize((640, 480), Image.ANTIALIAS)
    imgX1 = ImageTk.PhotoImage(image=imgX)
    panel = Label(root, image=imgX1)
    panel.image = imgX1
    panel.grid(row=2, column=2)
    return

# save image to directory
def save():
    image = imgX
    image.save("Q2.png")
    return

# Create a window
root = Tk()
# Set Title as Image Loader
root.title("Image Processing")
# Set the resolution of window
root.geometry("1920x1080")
# Allow Window to be resizable
root.resizable(width=True, height=True)
# Create a button and place it into the window using grid layout
btnOp = Button(root, text='Open', command=open_img).grid(row=1, column=1, padx=300, pady=10)
btnSave = Button(root, text='save', command=save).grid(row=1, column=2, padx=300, pady=10)

imag = Image.open('Q.jpg')
imag = imag.resize((640, 480), Image.ANTIALIAS)
imag = ImageTk.PhotoImage(imag)
panel1 = Label(root, image=imag)
panel2 = Label(root, image=imag)
panel1.image = imag
panel2.image = imag
panel1.grid(row=2, column=1)
panel2.grid(row=2, column=2)

# btnGray = Button(root, text='grayscale', command=gray).grid(row=3, column=1, padx=300, pady=10)
# btnInv = Button(root, text='Invert', command=invert).grid(row=4, column=1, padx=300, pady=10)
# btnGSlice = Button(root, text='Intensity Slicing', command=grayslice).grid(row=5, column=1, padx=300, pady=10)
# btnBPSlice = Button(root, text='Bit Plane Slicing', command=bitPlaneSlice).grid(row=5, column=1, padx=300, pady=10)
btnCS = Button(root, text='Contrast Stretch', command=contrastStretch).grid(row=6, column=1, padx=300, pady=10)
root.mainloop()


from data_structure import Node

class Solution(object):
    def hasCycle(self, head):
        """
        :type head: ListNode
        :rtype: bool
        """
        visited = set()
        while head:
            if head in visited:
                return True
            else:
                visited.add(head)
            head = head.next
        return False

    def hasCycleChasing(self,head):
        """
        :type head: ListNode
        :rtype: bool
        """
        fast = slow = head
        while slow and fast and fast.next:
            slow = slow.next
            fast = fast.next.next
            if slow is fast:
                return True
        return False

if __name__ == '__main__':
    n1 = Node(1)
    n2 = Node(2)
    n3 = Node(3)
    n1.next = n2
    n2.next = n3

    c1 = Node(1)
    c2 = Node(2)
    c3 = Node(3)
    c4 = Node(4)
    c1.next = c2
    c2.next = c3
    c3.next = c4
    c4.next = c2

    has_cycle = Solution().hasCycle(n1)
    assert(has_cycle is False)

    has_cycle = Solution().hasCycleChasing(c1)
    assert(has_cycle is True)


from __future__ import print_function, unicode_literals

import sys

reload(sys)
sys.setdefaultencoding('utf8')

import getopt
import json
import codecs
import os
import librosa

def read_audio_entries(data_file):
    json_data = json.load(codecs.open(data_file, 'r', 'utf-8'))
    audio_entries = list()
    for entry in json_data['fragments']:
        audio_entry = AudioEntry(entry)
        audio_entries.append(audio_entry)
    return audio_entries

def make_split(data_file, sound_file, start_shift_in_seconds=-0.1, end_shift_in_seconds=-0.1):
    audio_entries = read_audio_entries(data_file=data_file)
    result_path = os.path.splitext(sound_file)[0]
    if not os.path.exists(result_path):
        os.makedirs(result_path)

    return cut_sound_files_librosa(audio_entries, sound_file, result_path, start_shift_in_seconds, end_shift_in_seconds)

def cut_sound_files_librosa(audio_entries, sound_file, result_folder_path, start_shift_in_seconds=-0.1, end_shift_in_seconds=-0.1):
    audio_data, sample_rate = librosa.load(sound_file)
    for audio_entry in audio_entries:
        if len(audio_entry.text[0]) == 0:
            continue

        start = float(audio_entry.start)
        if start < 0:
            start = 0.0

        end = float(audio_entry.end)
        if end <= start:
            continue

        start_i = int(start * sample_rate)
        end_i = int(end * sample_rate)
        segment = audio_data[start_i:end_i]
        librosa.output.write_wav(os.path.join(result_folder_path, audio_entry.id) + '.wav', segment, sample_rate)
    return True

class AudioEntry:
    def __init__(self):
        self.start = None
        self.end = None
        self.id = None
        self.text = None

    def __init__(self, json_entry):
        self.start = json_entry['begin']
        self.end = json_entry['end']
        self.id = json_entry['id']
        self.text = json_entry['lines']

def print_usage():
    print('Usage:')
    print('\tpython audio_split.py -j <jsonfile> -a <audiofile>')
    sys.exit(1)

if __name__ == '__main__':
    try:
        options, arguments = getopt.getopt(sys.argv[1:], 'j:a:', ['--json', '--audio'])
    except getopt.GetoptError:
        print_usage()
    json_file = None
    audio_file = None

    for opt, arg in options:
        if opt in ('-j', '--json'):
            json_file = arg
        elif opt in ('-a', '--audio'):
            audio_file = arg

    if json_file is None or audio_file is None:
        print_usage()

    make_split(data_file=json_file, sound_file=audio_file)


# -*- coding: utf-8 -*-
'''
    归并排序:
    1.计算一个列表的中间位置，并且递归的排序其左边和右边的字列表（分而治之）
    2.将两个排好序的子列表重新合并为单个的排好序的列表
    3.当子列表不能够划分的时候，停止这个过程
'''

# 合并过程，用到一个额外的数组
from arrays import Array

def mergeSort(nums):
    copyBuffer = Array(len(nums))
    mergeSortHelper(nums,copyBuffer,0,len(nums) - 1)

def mergeSortHelper(nums,copyBuffer,left,right):
    if left < right:
        mid = left + (right-left)//2
        mergeSortHelper(nums,copyBuffer,left,mid)
        mergeSortHelper(nums,copyBuffer,mid+1,right)
        merge(nums,copyBuffer,left,mid,right)

def merge(nums,copyBuffer,left,mid,right):
    index1 = left
    index2 = mid + 1
    for i in range(left,right+1):
        if index1 > mid:
            copyBuffer[i] = nums[index2]
            index2 += 1
        elif index2 > right:
            copyBuffer[i] = nums[index1]
            index1 += 1
        elif nums[index1] < nums[index2]:
            copyBuffer[i] = nums[index1]
            index1 += 1
        else:
            copyBuffer[i] = nums[index2]
            index2 += 1

    for i in range(left,right+1):
        nums[i] = copyBuffer[i]


#Main application

import paranuara

app = paranuara.create_app()

# This is only used when running locally. When running live, gunicorn runs
# the application.
if __name__ == '__main__':
    app.run(host='127.0.0.1', port=8081, debug=True)


from unittest import TestCase

import textdistance
from gensim import matutils
from numpy.core.multiarray import dot

from model_util.entity_vector.entity_vector_model import EntityVectorModel
from script.knowledgeFusion.affinity_propagation import pre_process

class TestAffinityPropagation(TestCase):
    def test_pre_process(self):
        data_list = [{"id": 1, "sentence_id": 1, "paragraph_id": 1, "sentence": "just a test sentence",
                    "merge_np_id": 1, "noun_phrase": "test", "sub_nodes": []},
                     {"id": 2, "sentence_id": 2, "paragraph_id": 2, "sentence": "just a test sentence",
                      "merge_np_id": 1, "noun_phrase": "test", "sub_nodes": []},
                     {"id": 3, "sentence_id": 3, "paragraph_id": 3, "sentence": "just a test sentence",
                      "merge_np_id": 3, "noun_phrase": "test2", "sub_nodes": []},
                     {"id": 4, "sentence_id": 4, "paragraph_id": 4, "sentence": "test case",
                      "merge_np_id": 4, "noun_phrase": "test4", "sub_nodes": []},
                     {"id": 5, "sentence_id": 5, "paragraph_id": 5, "sentence": "just a test sentence",
                      "merge_np_id": 3, "noun_phrase": "test5", "sub_nodes": []}]
        result = pre_process(data_list)
        for each in result:
            print(each)
            print(len(each["sub_nodes"]))

    def test_similarity_calculation(self):
        str1 = "AbstractInputMethodService provides a abstract base class for inut methods."
        str2 = "The default implementation in this abstract class returns 1.0 for all components."

        vector_map = EntityVectorModel.load("mean_vector_api_paragraph.plain.txt", binary=False)

        vector1 = vector_map.compute_mean_vector(str1)
        vector2 = vector_map.compute_mean_vector(str2)
        semantic_similarity = dot(matutils.unitvec(vector1), matutils.unitvec(vector2))
        print("semantic similarity is " + semantic_similarity)
        structure_similarity = textdistance.jaccard(str1, str2)
        print("structure similarity is " + structure_similarity)


from flask.testing import FlaskClient

from main.models.movie import MovieModel
from main.models.phone import PhoneModel

sample_phone_uuid = "518361D5-B7DA-42D6-9CA6-025CB35BEF80"

class CustomClient(FlaskClient):
    def __init__(self, *args, **kwargs):
        self._headers = {
            "Authorization": f'Bearer {kwargs.pop("token")}',
        }

        super().__init__(*args, **kwargs)

    def _prepare_kwargs(self, kwargs):
        if "headers" in kwargs:
            kwargs["headers"].update(self._headers)
        else:
            kwargs["headers"] = self._headers

        return kwargs

    def get(self, *args, **kwargs):
        kwargs = self._prepare_kwargs(kwargs)

        return super().get(*args, **kwargs)

    def post(self, *args, **kwargs):
        kwargs = self._prepare_kwargs(kwargs)

        return super().post(*args, **kwargs)

    def put(self, *args, **kwargs):
        kwargs = self._prepare_kwargs(kwargs)

        return super().put(*args, **kwargs)

    def delete(self, *args, **kwargs):
        kwargs = self._prepare_kwargs(kwargs)

        return super().delete(*args, **kwargs)

def create_sample_phone(session) -> PhoneModel:
    phone = PhoneModel(uuid=sample_phone_uuid)
    session.add(phone)
    session.commit()
    return phone

def create_sample_movie(title: str, uuid: str, session) -> MovieModel:
    movie = MovieModel(
        title=title, description="a", uuid=uuid, release_year=1970, is_a_series=False
    )
    session.add(movie)
    session.commit()
    return movie


# -*- coding: utf-8 -*-
"""
Created on Sun Dec 31 20:42:47 2017

@author: zeba
"""

tup1 = (12, 34.56)
tup2 = ('abc', 'xyz')

# Following action is not valid for tuples because tuple in immutable.so after update we need to store the result in new variable
# tup1[0] = 100;

# So let's create a new tuple as follows
tup3 = tup1 + tup2
print (tup3)


"""
665. 平面范围求和 -不可变矩阵
给一 二维矩阵,计算由左上角 (row1, col1) 和右下角 (row2, col2) 划定的矩形内元素和.

样例
给出矩阵

[
  [3, 0, 1, 4, 2],
  [5, 6, 3, 2, 1],
  [1, 2, 0, 1, 5],
  [4, 1, 0, 1, 7],
  [1, 0, 3, 0, 5]
]
sumRegion(2, 1, 4, 3) -> 8
sumRegion(1, 1, 2, 2) -> 11
sumRegion(1, 2, 2, 4) -> 12

注意事项
你可以假设矩阵不变
对函数 sumRegion 的调用次数有很多次
你可以假设 row1 ≤ row2 并且 col1 ≤ col2
"""

# 方法一：Caching Rows
# 还记得我们使用累积和数组的1D版本吗？我们可以直接应用它来解决这个2D版本
# 尝试将2D矩阵视为m行的1D阵列。为了找到区域总和，我们只是逐行累积区域中的总和。
class NumMatrix:
    """
    @param: matrix: a 2D matrix
    """
    def __init__(self, matrix):
        # do intialization if necessary
        if len(matrix) == 0 or len(matrix[0]) == 0:
            return

        n = len(matrix)
        m = len(matrix[0])
        self.dp = [[0] * (m + 1) for _ in range(n + 1)]

        for r in range(n):
            for c in range(m):
                self.dp[r][c + 1] = self.dp[r][c] + matrix[r][c]

    """
    @param: row1: An integer
    @param: col1: An integer
    @param: row2: An integer
    @param: col2: An integer
    @return: An integer
    """
    def sumRegion(self, row1, col1, row2, col2):
        # write your code here
        ans = 0
        for i in range(row1, row2 + 1):
            ans += self.dp[i][col2 + 1] - self.dp[i][col1]
        return ans

# 方法二：Caching Smarter
# 我们在1D版本中使用了累积和数组。 我们注意到累积和是相对于索引0处的原点计算的。
# 将这个类比扩展到2D情况，我们可以预先计算相对于原点的累积区域和(0,0)。
# sum(ABCD) = sum(OD) - sum(OB) - sum(OC) + sum(OA)
class NumMatrix1:
    """
    @param: matrix: a 2D matrix
    """
    def __init__(self, matrix):
        # do intialization if necessary
        if len(matrix) == 0 or len(matrix[0]) == 0:
            return

        n = len(matrix)
        m = len(matrix[0])
        self.dp = [[0] * (m + 1) for _ in range(n + 1)]

        for r in range(n):
            for c in range(m):
                self.dp[r + 1][c + 1] = self.dp[r + 1][c] + self.dp[r][c + 1] + matrix[r][c] - self.dp[r][c]

    """
    @param: row1: An integer
    @param: col1: An integer
    @param: row2: An integer
    @param: col2: An integer
    @return: An integer
    """
    def sumRegion(self, row1, col1, row2, col2):
        # write your code here
        return self.dp[row2 + 1][col2 + 1] - self.dp[row1][col2 + 1] - self.dp[row2 + 1][col1] + self.dp[row1][col1]

# Your NumMatrix object will be instantiated and called as such:
# obj = NumMatrix(matrix)
# param_1 = obj.sumRegion(row1,col1,row2,col2)

if __name__ == '__main__':
    nm = NumMatrix1([
        [3, 0, 1, 4, 2],
        [5, 6, 3, 2, 1],
        [1, 2, 0, 1, 5],
        [4, 1, 0, 1, 7],
        [1, 0, 3, 0, 5]
    ])
    print(nm.sumRegion(2, 1, 4, 3))
    print(nm.sumRegion(1, 1, 2, 2))
    print(nm.sumRegion(1, 2, 2, 4))


import multiprocessing
import numpy as np
import pandas as pd
from pathlib import Path
import pickle

import replays_to_database

def process_replay_folder(replay_folder):
    replay_paths = list(replay_folder.glob('*[0-9].json'))
    sub_id = int(replay_folder.name)
    print(f'Converting {len(replay_paths)} replays to visualization format for submission {sub_id} in {replay_folder}')

    n_replays_processed = 0
    for rp in replay_paths:
        # if not Path(f'{replay_folder}/{rp.stem}_visualization_dict.pkl').exists() or FORCE:
        replay = replays_to_database.read_json(rp)
        replay_info = replays_to_database.read_json(rp.parent / (rp.stem + '_info.json'))
        if replay_info['agents'][0]['submissionId'] == sub_id:
            agent_idx = 0
        elif replay_info['agents'][1]['submissionId'] == sub_id:
            agent_idx = 1
        else:
            raise RuntimeError(f'{rp}: Neither agent has the correct submissionId')
        replay_viz_dict = replays_to_database.convert_replay_for_visualization(replay, agent_idx)

        if replay_viz_dict is not None:
            with open(f'{replay_folder}/{rp.stem}_visualization_dict.pkl', 'wb') as f:
                pickle.dump(replay_viz_dict, f, pickle.HIGHEST_PROTOCOL)
            n_replays_processed += 1

    return n_replays_processed

if __name__ == '__main__':
    DESTINATION = 'episode_scraping/replays_by_sub_id'
    sub_id_to_model_name_df = pd.read_csv(f'{DESTINATION}/sub_id_to_model_name.csv', index_col='sub_id')
    nan_mask = sub_id_to_model_name_df.index.isna() | np.any(sub_id_to_model_name_df.isna(), axis=1)
    if nan_mask.any():
        print('Ignoring the following nan-valued rows:')
        sub_id_to_model_name_df = sub_id_to_model_name_df[~nan_mask]
        sub_id_to_model_name_df.index = sub_id_to_model_name_df.index.astype(int)
    sub_id_to_model_names_dict = sub_id_to_model_name_df.to_dict()['model_name']
    assert len(np.unique(list(sub_id_to_model_name_df.index))) == len(sub_id_to_model_name_df)

    replay_folders = [Path(f'{DESTINATION}/{sub_id}') for sub_id in sub_id_to_model_names_dict.keys()]
    with multiprocessing.Pool(processes=multiprocessing.cpu_count() - 1) as pool:
        n_replays = pool.map(process_replay_folder, replay_folders)

    print(f'Finished processing {np.sum(n_replays)} replays from {len(replay_folders)} submissions')


#!/usr/bin/env python
import math
import matplotlib.pyplot as plt

def distance(p1, p2):
    return math.sqrt((p2[0] - p1[0]) ** 2 + (p2[1] - p1[1]) ** 2)

def distance2(p1, p2):
    return (p2[0] - p1[0]) ** 2 + (p2[1] - p1[1]) ** 2

def k_means(seeds, points, distance_fn):
    results = [[] for _ in range(len(seeds))]

    for point in points:
        min_dist = math.inf
        min_element = 0
        for i in range(len(seeds)):
            new_dist = distance_fn(seeds[i], point)
            if new_dist < min_dist:
                min_dist = new_dist
                min_element = i
        (results[min_element]).append(point)

    new_seeds = seeds[:]
    for i in range(len(seeds)):
        cluster = results[i]
        if len(cluster) < 1:
            continue
        center = seeds[i]
        distance_sum = (0, 0)
        for point in cluster:
            dist = (center[0] - point[0], center[1] - point[1])
            distance_sum = (distance_sum[0] + dist[0], distance_sum[1] + dist[1])
        mean = (distance_sum[0] / len(cluster), distance_sum[1] / len(cluster))
        new_seeds[i] = (new_seeds[i][0] - mean[0], new_seeds[i][1] - mean[1])

    return results, new_seeds

def plot_result(result, seeds):
    x_list = [list()] * len(result)
    y_list = [list()] * len(result)

    for i in range(len(x_list)):
        x_list[i] = [p[0] for p in result[i]]
        y_list[i] = [p[1] for p in result[i]]

    for i in range(len(x_list)):
        plt.plot(x_list[i], y_list[i], 'o')

    for seed in seeds:
        plt.plot(seed[0], seed[1], 'o')

    plt.xlim((-1, 6))
    plt.ylim((0, 3))
    plt.show()

seeds = [(1, 1.5), (3, 1)]
points = [
    (0, 0.8), (0, 0.5), (1.2, 0.4), (1.5, 0.8), (1, 1), (1.5, 1),
    (2.5, 1), (3, 1), (3, 2), (4, 1.5), (4, 2.5), (5, 2)
]

regular_result, new_seeds = k_means(seeds, points, distance)
plot_result(regular_result, seeds)

regular_result, _ = k_means(new_seeds, points, distance)
plot_result(regular_result, new_seeds)


import torch
from scripts.utils import *
from tqdm import tqdm

def train_model(model, optimizer, loss_function,
                data_dict, batch_size, word_indexer,
                tag_indexer, training_metrics, validation_metrics, num_epochs=5):
    val_words, val_tags = data_dict['dev']

    def validate_model():
        with torch.no_grad():
            inputs = torch.tensor(word_indexer.elements_to_index(val_words), dtype=torch.long)
            true_vals = tag_indexer.elements_to_index(val_tags)
            tag_scores = model(inputs)
            prediction = get_tag_indexes_from_scores(tag_scores)
        validation_metrics.update(prediction, true_vals)
        validation_metrics.collect()
        for metric in validation_metrics.metrics_dict.keys():
            print(f"{metric} - {validation_metrics.metrics_dict[metric][-1]}")
        print()

    words, tags = data_dict['train']

    batches = list(get_batched(words, tags, batch_size))

    losses = []

    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch + 1, num_epochs))
        print('-' * 10)

        running_loss = 0.0

        for sentence, tags in tqdm(batches):
            model.zero_grad()

            sentence_in = torch.tensor(word_indexer.elements_to_index(sentence), dtype=torch.long)
            targets = torch.tensor(tag_indexer.elements_to_index(tags), dtype=torch.long)

            tag_scores = model(sentence_in)

            loss = loss_function(tag_scores, targets)
            loss.backward()
            optimizer.step()

            prediction = get_tag_indexes_from_scores(tag_scores.detach().numpy())
            training_metrics.update(prediction, targets)

            running_loss += loss.item() * sentence_in.size(0)

        training_metrics.collect()
        epoch_loss = running_loss / len(batches)

        losses.append(epoch_loss)
        print(f"Loss per epoch - {epoch_loss}")
        for metric in training_metrics.metrics_dict.keys():
            print(f"{metric} - {training_metrics.metrics_dict[metric][-1]}")
        print()
        print("Validating on dev test: ")
        validate_model()
        print()

    return model, training_metrics, validation_metrics, losses


"""
460. LFU Cache
Hard

1959

151

Add to List

Share
Design and implement a data structure for a Least Frequently Used (LFU) cache.

Implement the LFUCache class:

LFUCache(int capacity) Initializes the object with the capacity of the data structure.
int get(int key) Gets the value of the key if the key exists in the cache. Otherwise, returns -1.
void put(int key, int value) Update the value of the key if present, or inserts the key if not already present. When the cache reaches its capacity, it should invalidate and remove the least frequently used key before inserting a new item. For this problem, when there is a tie (i.e., two or more keys with the same frequency), the least recently used key would be invalidated.
To determine the least frequently used key, a use counter is maintained for each key in the cache. The key with the smallest use counter is the least frequently used key.

When a key is first inserted into the cache, its use counter is set to 1 (due to the put operation). The use counter for a key in the cache is incremented either a get or put operation is called on it.

Example 1:

Input
["LFUCache", "put", "put", "get", "put", "get", "get", "put", "get", "get", "get"]
[[2], [1, 1], [2, 2], [1], [3, 3], [2], [3], [4, 4], [1], [3], [4]]
Output
[null, null, null, 1, null, -1, 3, null, -1, 3, 4]

Explanation
// cnt(x) = the use counter for key x
// cache=[] will show the last used order for tiebreakers (leftmost element is  most recent)
LFUCache lfu = new LFUCache(2);
lfu.put(1, 1);   // cache=[1,_], cnt(1)=1
lfu.put(2, 2);   // cache=[2,1], cnt(2)=1, cnt(1)=1
lfu.get(1);      // return 1
                 // cache=[1,2], cnt(2)=1, cnt(1)=2
lfu.put(3, 3);   // 2 is the LFU key because cnt(2)=1 is the smallest, invalidate 2.
                 // cache=[3,1], cnt(3)=1, cnt(1)=2
lfu.get(2);      // return -1 (not found)
lfu.get(3);      // return 3
                 // cache=[3,1], cnt(3)=2, cnt(1)=2
lfu.put(4, 4);   // Both 1 and 3 have the same cnt, but 1 is LRU, invalidate 1.
                 // cache=[4,3], cnt(4)=1, cnt(3)=2
lfu.get(1);      // return -1 (not found)
lfu.get(3);      // return 3
                 // cache=[3,4], cnt(4)=1, cnt(3)=3
lfu.get(4);      // return 4
                 // cache=[3,4], cnt(4)=2, cnt(3)=3

Constraints:

0 <= capacity, key, value <= 104
At most 105 calls will be made to get and put.

Follow up: Could you do both operations in O(1) time complexity?

"""
"""
Design

use nums: {key: freq}
use freqs: {freq: OrderedDict {key: value}}

key points:
1. OrderedDict orders item by insert order
2. OrderedDict.move_to_end(key) moves an key to end (re-insert)
3. OrderedDict.popitem(last=False) removes oldest value (head)

"""
from collections import defaultdict, OrderedDict
class LFUCache:

    def __init__(self, capacity: int):
        self.size = 0  # total number of caches
        self.capacity = capacity
        self.nums = {} # key to freq
        self.freqs = defaultdict(OrderedDict)
        self.minfreq = 0  # the minimum freq, to help identify which dll to evict an cached item

    def _update(self, key, value=None):
        """
        update node when add(key) or put(key) and the key exists already
        1. remove node from self.freq[node.freq] doublylinkedlist
        2. update node.freq node.freq += 1
        3. add node to self.freq[node.freq] doublylinkedlist (this is old freq+1)
        4. if minfreq = old freq, and self.freqs[old freq].dllsize == 0: update minfreq to new freq = old freq + 1
        """
        freq = self.nums[key]
        # remove key from self.freqs[freq]
        self.freqs[freq].move_to_end(key)
        item = self.freqs[freq].popitem(last=True)
        newfreq = freq + 1
        # check minfreq
        if freq == self.minfreq and len(self.freqs[freq]) == 0:  # minfreq list is now empty
            self.minfreq = newfreq  # we know this node would move from freq to freq+1, so minfreq += 1
        # add key into self.freqs[newfreq]
        self.nums[key] = newfreq
        if value is not None:
            self.freqs[newfreq][key] = value # update to new value
        else:
            self.freqs[newfreq][key] = item[1] # dont' update value

    def get(self, key: int) -> int:
        # print('get key=%s self.nums=%s self.freqs=%s' % (key, self.nums, self.freqs))
        if key not in self.nums:
            # print('get key=%s returning %s' % (key, -1))
            return -1
        self._update(key, value=None)
        # print('get key=%s returning %s' % (key, self.freqs[self.nums[key]][key]))
        return self.freqs[self.nums[key]][key]

    def put(self, key: int, value: int) -> None:
        """
        if key exist, update freq and value in OrderedDict
        """
        # print('put key=%s value=%s self.nums=%s self.freqs=%s' % (key, value, self.nums, self.freqs))
        if self.capacity == 0:
            return
        if key in self.nums:
            self._update(key, value)
        else:
            # new key, evict one if cache full
            if self.size == self.capacity:
                # remove least frequently used item in self.freqs[self.minfreq]
                item = self.freqs[self.minfreq].popitem(last=False)
                del self.nums[item[0]]
                self.size -= 1
            self.freqs[1][key] = value
            self.size += 1
            self.nums[key] = 1
            self.minfreq = 1

"""
Design

use map to store key to node
dict of frequency {freq: doublylinkedlist of LRU of same freq}

key points:
1. self.minfreq keep track of minfreq, so when evicting least frequenty used, we can get it via self.freqs[self.minfreq]
2. keep freq inside node
3. keep doublylinkedlist size inside doublylinkedlist
4. keep total cache size in LFUCache self.size (to check capacity easily)
5. check capacity before adding new node
6. when updating node, move it from old freq list to new list, we need to update minfreq

mistakes:
1. use LFUCache.size instead of len(self.nums) to keep track of exceeding capacity or not

"""

class DLinkedNode:
    def __init__(self, key=-1, val=-1, prev=None, nxt=None):
        # use key=-1 and val=-1 to indicate dummy node
        self.key = key
        self.val = val
        self.prev = prev
        self.nxt = nxt
        self.freq = 1

    def __repr__(self):
        return '{%s : %s (%s)}' % (self.key, self.val, self.freq)

class DLinkedList:
    """
    represent a Least Recently Used (LRU) list, excluding dummy head/tail, the first element is the most recently used
    when expiring least recently used, remove this.tail.prev
    """

    def __init__(self):
        self.head = DLinkedNode()
        self.tail = DLinkedNode()
        self.head.nxt = self.tail
        self.tail.prev = self.head
        self.dllsize = 0

    def add_node(self, node):
        """
        add node to right after dummy head
        """
        node.prev, node.nxt = self.head, self.head.nxt
        self.head.nxt.prev = node
        self.head.nxt = node
        self.dllsize += 1

    def remove_node(self, node):
        node_prev = node.prev
        node_nxt = node.nxt
        node_prev.nxt = node_nxt
        node_nxt.prev = node_prev
        self.dllsize -= 1

        return node

    def evict_lru(self):
        """
        evict the item with least recent use time for given freq
        """
        return self.remove_node(self.tail.prev)

    def move_to_head(self, node):
        """
        move a node to head (just used, updating to most recently used)
        """
        node = self.remove_node(node)
        self.add_node(node)

    def __repr__(self):
        ans = ''
        cur = self.head.nxt
        while cur != self.tail:
            ans +=  str(cur)
            cur = cur.nxt

        return ans

class LFUCache1:

    def __init__(self, capacity: int):
        self.size = 0  # total number of caches
        self.capacity = capacity
        self.nums = {}
        self.freqs = defaultdict(DLinkedList)
        self.minfreq = 0  # the minimum freq, to help identify which dll to evict an cached item

    def _update(self, node):
        """
        update node when add(key) or put(key) and the key exists already
        1. remove node from self.freq[node.freq] doublylinkedlist
        2. update node.freq node.freq += 1
        3. add node to self.freq[node.freq] doublylinkedlist (this is old freq+1)
        4. if minfreq = old freq, and self.freqs[old freq].dllsize == 0: update minfreq to new freq = old freq + 1
        """
        freq = node.freq
        self.freqs[freq].remove_node(node)
        newfreq = freq + 1
        node.freq = newfreq
        # check minfreq
        if freq == self.minfreq and self.freqs[freq].dllsize == 0:  # minfreq list is now empty
            self.minfreq = newfreq  # we know this node would move from freq to freq+1, so minfreq += 1
        if newfreq not in self.freqs:
            self.freqs[newfreq] = DLinkedList()
        self.freqs[newfreq].add_node(node)

    def get(self, key: int) -> int:
        # print('get key=%s self.nums=%s self.freqs=%s' % (key, self.nums, self.freqs))
        if key not in self.nums:
            print('get key=%s returning %s' % (key, -1))
            return -1
        node = self.nums[key]
        self._update(node)
        print('get key=%s returning %s' % (key, node.val))
        return node.val

    def put(self, key: int, value: int) -> None:
        """
        if key exist, update node value, and update node
        """
        # print('put key=%s value=%s self.nums=%s self.freqs=%s' % (key, value, self.nums, self.freqs))
        if self.capacity == 0:
            return
        if key in self.nums:
            node = self.nums[key]
            node.val = value
            self._update(node)
        else:
            # new key, evict one if cache full
            if self.size == self.capacity:
                # remove last in doublylinkedlist in the smallest frequency
                node = self.freqs[self.minfreq].evict_lru()
                del self.nums[node.key]
                self.size -= 1
            node = DLinkedNode(key=key, val=value)
            if 1 not in self.freqs:
                self.freqs[1] = DLinkedList()
            self.freqs[1].add_node(node)
            self.size += 1
            self.nums[key] = node
            self.minfreq = 1

# Your LFUCache object will be instantiated and called as such:
# obj = LFUCache(capacity)
# param_1 = obj.get(key)
# obj.put(key,value)

def main():
    obj = LFUCache(2)
    obj.put(1,1)
    obj.put(2,2)
    assert obj.get(1) == 1, 'fails'
    obj.put(3,3)
    assert obj.get(2) == -1, 'fails'
    assert obj.get(3) == 3, 'fails'
    obj.put(4,4)
    assert obj.get(1) == -1, 'fails'
    assert obj.get(3) == 3, 'fails'
    assert obj.get(4) == 4, 'fails'

    # obj = LFUCache(0)
    # obj.put(0,0)
    # assert obj.get(0) == -1, 'fails'

if __name__ == '__main__':
   main()


import pygame
from pygame.locals import *
from OpenGL.GL import *
from OpenGL.GLU import *
from figure import *
from tkinter import *
from tkinter import ttk
import time
import serial
import threading
from dataPlotter import live_plotter

ApplicationGL = False

class PortSettings:
    Name = "COM3"
    Speed = 115200
    Timeout = 2

class IMU:
    Roll = 0
    Pitch = 0
    Yaw = 0

myport = PortSettings()
myimu  = IMU()

def RunAppliction():
    global ApplicationGL
    myport.Name = Port_entry.get()
    myport.Speed = Baud_entry.get()
    ApplicationGL = True
    ConfWindw.destroy()

ConfWindw = Tk()
ConfWindw.title("Configure Serial Port")
ConfWindw.configure(bg = "#2E2D40")
ConfWindw.geometry('300x150')
ConfWindw.resizable(width=False, height=False)
positionRight = int(ConfWindw.winfo_screenwidth()/2 - 300/2)
positionDown = int(ConfWindw.winfo_screenheight()/2 - 150/2)
ConfWindw.geometry("+{}+{}".format(positionRight, positionDown))

Port_label = Label(text = "Port:", font =("",12), justify= "right", bg = "#2E2D40", fg = "#FFFFFF")
Port_label.place(x = 50, y =30, anchor = "center")
Port_entry = Entry(width = 20, bg = "#37364D", fg = "#FFFFFF", justify = "center")
Port_entry.insert(INSERT,myport.Name)
Port_entry.place(x = 180, y = 30,anchor = "center")

Baud_label = Label(text = "Speed:", font =("",12), justify= "right", bg = "#2E2D40", fg = "#FFFFFF")
Baud_label.place(x = 50, y =80, anchor = "center")
Baud_entry = Entry(width = 20,bg = "#37364D", fg = "#FFFFFF", justify = "center")
Baud_entry.insert(INSERT,str(myport.Speed))
Baud_entry.place(x = 180, y = 80, anchor = "center")

ok_button = Button(text = "Ok",width = 8,command = RunAppliction,bg="#135EF2",fg ="#FFFFFF")
ok_button.place(x = 150, y = 120, anchor="center")

def InitPygame():
    global display
    #global gameDisplay
    pygame.init()
    display = (1280,960)
    #gameDisplay =  pygame.display.set_mode(display, DOUBLEBUF|OPENGL)
    pygame.display.set_mode(display, DOUBLEBUF|OPENGL)
    pygame.display.set_caption('IMU visualizer   (Press Esc to exit)')

def InitGL():
    glClearColor((1.0/255*46),(1.0/255*45),(1.0/255*64),1)
    glEnable(GL_DEPTH_TEST)
    glDepthFunc(GL_LEQUAL)
    glHint(GL_PERSPECTIVE_CORRECTION_HINT, GL_NICEST)

    gluPerspective(100, (display[0]/display[1]), 0.1, 50.0)
    glTranslatef(0.0,0.0, -5)

def DrawText(textString):
    font = pygame.font.SysFont ("Courier New",25, True)
    textSurface = font.render(textString, True, (255,255,0), (46,45,64,255))
    textData = pygame.image.tostring(textSurface, "RGBA", True)
    glDrawPixels(textSurface.get_width(), textSurface.get_height(), GL_RGBA, GL_UNSIGNED_BYTE, textData)

def DrawBoard():

    glBegin(GL_QUADS)
    x = 0

    for surface in surfaces:

        for vertex in surface:
            glColor3fv((colors[x]))
            glVertex3fv(vertices[vertex])
        x += 1
    glEnd()

def DrawGL():

    glClear(GL_COLOR_BUFFER_BIT|GL_DEPTH_BUFFER_BIT)
    glLoadIdentity()
    gluPerspective(90, (display[0]/display[1]), 0.1, 50.0)
    glTranslatef(0.0,0.0, -5)

    glRotatef(round(myimu.Pitch,1), 0, 0, 1)
    glRotatef(round(myimu.Roll,1), -1, 0, 0)
    glRotatef(round(myimu.Yaw,1), 0, 1, 0)

    DrawText(" Roll: {}°     Pitch: {}°      Yaw: {}°".format(round(myimu.Roll,1),round(myimu.Pitch,1),round(myimu.Yaw,1)))
    DrawBoard()

    # plotImg = pygame.image.load('plot.png')

    # imgData = pygame.image.tostring(plotImg, "RGBA", True)

    # glDrawPixels(plotImg.get_width(), plotImg.get_height(), GL_RGBA, GL_UNSIGNED_BYTE, imgData)

    #gameDisplay.blit(plotImg, (300,300))

    pygame.display.flip()

def SerialConnection ():
    global serial_object
    serial_object = serial.Serial( myport.Name, baudrate= myport.Speed, timeout = myport.Timeout)

class ReadLine:
    def __init__(self, s):
        self.buf = bytearray()
        self.s = s

    def readline(self):
        i = self.buf.find(b"\n")
        if i >= 0:
            r = self.buf[:i+1]
            self.buf = self.buf[i+1:]
            return r
        while True:
            i = max(1, min(2048, self.s.in_waiting))
            data = self.s.read(i)
            i = data.find(b"\n")
            if i >= 0:
                r = self.buf + data[:i+1]
                self.buf[0:] = data[i+1:]
                return r
            else:
                self.buf.extend(data)

def ReadData():

    rl = ReadLine(serial_object)

    while True:

        data = rl.readline().decode('unicode_escape')
        #data = serial_object.read(256)
        #data.splitlines()

        # Received dataframe example:
        # Orientation: 347.66, 1.26, 1.38 (YAW, PITCH, ROLL)
        # Quaternion: 0.1076, -0.0097, 0.0132, 0.9941

        if data[0] == 'O': #Orientation
            data = data[13:].rstrip('\n')
            orientationData = data.split(', ')
            yaw = orientationData[0]
            pitch = orientationData[1]
            roll = orientationData[2]

            print(yaw, pitch, roll)

            #Convert string values to float
            myimu.Roll = float(roll)
            myimu.Pitch = float(pitch)
            myimu.Yaw = float(yaw)

def main():
    ConfWindw.mainloop()
    if ApplicationGL == True:
        InitPygame()
        InitGL()

        try:
            SerialConnection()
            line1 = []
            line2 = []
            line3 = []
            list = [0] * 100
            list2 = [0] * 100
            list3 = [0] * 100

            pygame.time.wait(10)
            myThread1 = threading.Thread(target = ReadData)
            myThread1.daemon = True
            myThread1.start()
            while True:
                event = pygame.event.poll()
                line1, line2, line3, list, list2, list3 = live_plotter(myimu.Yaw, myimu.Pitch, myimu.Roll, line1, line2, line3, list, list2, list3)
                if event.type == QUIT or (event.type == KEYDOWN and event.key == K_ESCAPE):
                    pygame.quit()
                    break

                DrawGL()
                pygame.time.wait(10)

        except:
            glClear(GL_COLOR_BUFFER_BIT|GL_DEPTH_BUFFER_BIT)
            DrawText("Sorry, something is wrong :c")
            pygame.display.flip()
            time.sleep(5)

if __name__ == '__main__': main()


from flask import render_template, request, jsonify

from . import storage, VERSION

def todo_page():
    data = {"VERSION": VERSION}
    return render_template("service/todo.html", **data)

def todo_json_get():
    items = storage.Todo.select()

    data = []
    for item in items:
        row = {
            "id": item.id,
            "title": item.title,
            "description": item.description,
            "created": str(item.created),
            "updated": str(item.updated),
            "status": item.status,
            "priority": item.priority
        }
        data.append(row)

    return jsonify(items=data)

def todo_json_add():

    in_data = request.get_json()
    if in_data is None:
        raise Exception('no input data')

    storage.Todo.create(title=in_data["title"],
                        description=in_data["description"],
                        status=in_data["status"],
                        priority=in_data["priority"])
    last_record = storage.Todo.select().order_by(storage.Todo.id.desc()).get()

    in_data["id"] = last_record.id
    in_data["created"] = str(last_record.created)
    in_data["updated"] = str(last_record.updated)
    return jsonify(item=in_data)

def todo_json_update():

    in_data = request.get_json()

    if in_data is None:
        raise Exception('no input data')

    item = storage.Todo.get(storage.Todo.id == in_data["id"])
    item.title = in_data["title"]
    item.description = in_data["description"]
    item.status = in_data["status"]
    item.priority = in_data["priority"]
    item.updated = storage.sql_date()
    item.save()

    in_data["updated"] = str(item.updated)

    return jsonify(item=in_data)

def todo_json_remove():

    in_data = request.get_json()

    if in_data is None:
        raise Exception('no input data')

    item = storage.Todo.get(storage.Todo.id == in_data["id"])
    item.delete_instance()

    return jsonify(item=in_data)


__all__ = ["ProcessCcdWithFakesConfig", "ProcessCcdWithFakesTask",
           "ProcessCcdWithVariableFakesConfig", "ProcessCcdWithVariableFakesTask"]

import numpy as np
import pandas as pd

import lsst.pex.config as pexConfig
import lsst.pipe.base as pipeBase

from .insertFakes import InsertFakesTask
from lsst.afw.table import SourceTable
from lsst.meas.base import IdGenerator, DetectorVisitIdGeneratorConfig
from lsst.pipe.base import PipelineTask, PipelineTaskConfig, PipelineTaskConnections
import lsst.pipe.base.connectionTypes as cT
import lsst.afw.table as afwTable
from lsst.skymap import BaseSkyMap
from lsst.pipe.tasks.calibrate import CalibrateTask

class ProcessCcdWithFakesConnections(PipelineTaskConnections,
                                     dimensions=("instrument", "visit", "detector"),
                                     defaultTemplates={"coaddName": "deep",
                                                       "wcsName": "gbdesAstrometricFit",
                                                       "photoCalibName": "jointcal",
                                                       "fakesType": "fakes_"}):
    skyMap = cT.Input(
        doc="Input definition of geometry/bbox and projection/wcs for "
        "template exposures. Needed to test which tract to generate ",
        name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
        dimensions=("skymap",),
        storageClass="SkyMap",
    )

    exposure = cT.Input(
        doc="Exposure into which fakes are to be added.",
        name="calexp",
        storageClass="ExposureF",
        dimensions=("instrument", "visit", "detector")
    )

    fakeCats = cT.Input(
        doc="Set of catalogs of fake sources to draw inputs from. We "
            "concatenate the tract catalogs for detectorVisits that cover "
            "multiple tracts.",
        name="{fakesType}fakeSourceCat",
        storageClass="DataFrame",
        dimensions=("tract", "skymap"),
        deferLoad=True,
        multiple=True,
    )

    externalSkyWcsTractCatalog = cT.Input(
        doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
             "id for the catalog id, sorted on id for fast lookup."),
        name="{wcsName}SkyWcsCatalog",
        storageClass="ExposureCatalog",
        dimensions=("instrument", "visit", "tract", "skymap"),
        deferLoad=True,
        multiple=True,
    )

    externalSkyWcsGlobalCatalog = cT.Input(
        doc=("Per-visit wcs calibrations computed globally (with no tract information). "
             "These catalogs use the detector id for the catalog id, sorted on id for "
             "fast lookup."),
        name="finalVisitSummary",
        storageClass="ExposureCatalog",
        dimensions=("instrument", "visit"),
    )

    externalPhotoCalibTractCatalog = cT.Input(
        doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
             "detector id for the catalog id, sorted on id for fast lookup."),
        name="{photoCalibName}PhotoCalibCatalog",
        storageClass="ExposureCatalog",
        dimensions=("instrument", "visit", "tract"),
        deferLoad=True,
        multiple=True,
    )

    externalPhotoCalibGlobalCatalog = cT.Input(
        doc=("Per-visit photometric calibrations. These catalogs use the "
             "detector id for the catalog id, sorted on id for fast lookup."),
        name="finalVisitSummary",
        storageClass="ExposureCatalog",
        dimensions=("instrument", "visit"),
    )

    icSourceCat = cT.Input(
        doc="Catalog of calibration sources",
        name="icSrc",
        storageClass="SourceCatalog",
        dimensions=("instrument", "visit", "detector")
    )

    sfdSourceCat = cT.Input(
        doc="Catalog of calibration sources",
        name="src",
        storageClass="SourceCatalog",
        dimensions=("instrument", "visit", "detector")
    )

    outputExposure = cT.Output(
        doc="Exposure with fake sources added.",
        name="{fakesType}calexp",
        storageClass="ExposureF",
        dimensions=("instrument", "visit", "detector")
    )

    outputCat = cT.Output(
        doc="Source catalog produced in calibrate task with fakes also measured.",
        name="{fakesType}src",
        storageClass="SourceCatalog",
        dimensions=("instrument", "visit", "detector"),
    )

    def __init__(self, *, config=None):
        super().__init__(config=config)

        if not config.doApplyExternalGlobalPhotoCalib:
            self.inputs.remove("externalPhotoCalibGlobalCatalog")
        if not config.doApplyExternalTractPhotoCalib:
            self.inputs.remove("externalPhotoCalibTractCatalog")

        if not config.doApplyExternalGlobalSkyWcs:
            self.inputs.remove("externalSkyWcsGlobalCatalog")
        if not config.doApplyExternalTractSkyWcs:
            self.inputs.remove("externalSkyWcsTractCatalog")

class ProcessCcdWithFakesConfig(PipelineTaskConfig,
                                pipelineConnections=ProcessCcdWithFakesConnections):
    """Config for inserting fake sources

    Notes
    -----
    The default column names are those from the UW sims database.
    """

    doApplyExternalGlobalPhotoCalib = pexConfig.Field(
        dtype=bool,
        default=False,
        doc="Whether to apply an external photometric calibration via an "
            "`lsst.afw.image.PhotoCalib` object. Uses the "
            "`externalPhotoCalibName` config option to determine which "
            "calibration to use. Uses a global calibration."
    )

    doApplyExternalTractPhotoCalib = pexConfig.Field(
        dtype=bool,
        default=False,
        doc="Whether to apply an external photometric calibration via an "
            "`lsst.afw.image.PhotoCalib` object. Uses the "
            "`externalPhotoCalibName` config option to determine which "
            "calibration to use. Uses a per tract calibration."
    )

    externalPhotoCalibName = pexConfig.ChoiceField(
        doc="What type of external photo calib to use.",
        dtype=str,
        default="jointcal",
        allowed={"jointcal": "Use jointcal_photoCalib",
                 "fgcm": "Use fgcm_photoCalib",
                 "fgcm_tract": "Use fgcm_tract_photoCalib"}
    )

    doApplyExternalGlobalSkyWcs = pexConfig.Field(
        dtype=bool,
        default=False,
        doc="Whether to apply an external astrometric calibration via an "
            "`lsst.afw.geom.SkyWcs` object. Uses the "
            "`externalSkyWcsName` config option to determine which "
            "calibration to use. Uses a global calibration."
    )

    doApplyExternalTractSkyWcs = pexConfig.Field(
        dtype=bool,
        default=False,
        doc="Whether to apply an external astrometric calibration via an "
            "`lsst.afw.geom.SkyWcs` object. Uses the "
            "`externalSkyWcsName` config option to determine which "
            "calibration to use. Uses a per tract calibration."
    )

    externalSkyWcsName = pexConfig.ChoiceField(
        doc="What type of updated WCS calib to use.",
        dtype=str,
        default="gbdesAstrometricFit",
        allowed={"gbdesAstrometricFit": "Use gbdesAstrometricFit_wcs"}
    )

    coaddName = pexConfig.Field(
        doc="The name of the type of coadd used",
        dtype=str,
        default="deep",
    )

    srcFieldsToCopy = pexConfig.ListField(
        dtype=str,
        default=("calib_photometry_reserved", "calib_photometry_used", "calib_astrometry_used",
                 "calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
        doc=("Fields to copy from the `src` catalog to the output catalog "
             "for matching sources Any missing fields will trigger a "
             "RuntimeError exception.")
    )

    matchRadiusPix = pexConfig.Field(
        dtype=float,
        default=3,
        doc=("Match radius for matching icSourceCat objects to sourceCat objects (pixels)"),
    )

    doMatchVisit = pexConfig.Field(
        dtype=bool,
        default=False,
        doc="Match visit to trim the fakeCat"
    )

    calibrate = pexConfig.ConfigurableField(target=CalibrateTask,
                                            doc="The calibration task to use.")

    insertFakes = pexConfig.ConfigurableField(target=InsertFakesTask,
                                              doc="Configuration for the fake sources")

    idGenerator = DetectorVisitIdGeneratorConfig.make_field()

    def setDefaults(self):
        super().setDefaults()
        self.calibrate.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("FAKE")
        self.calibrate.measurement.plugins["base_PixelFlags"].masksFpCenter.append("FAKE")
        self.calibrate.doAstrometry = False
        self.calibrate.doWriteMatches = False
        self.calibrate.doPhotoCal = False
        self.calibrate.doComputeSummaryStats = False
        self.calibrate.detection.reEstimateBackground = False

class ProcessCcdWithFakesTask(PipelineTask):
    """Insert fake objects into calexps.

    Add fake stars and galaxies to the given calexp, specified in the dataRef. Galaxy parameters are read in
    from the specified file and then modelled using galsim. Re-runs characterize image and calibrate image to
    give a new background estimation and measurement of the calexp.

    `ProcessFakeSourcesTask` inherits six functions from insertFakesTask that make images of the fake
    sources and then add them to the calexp.

    `addPixCoords`
        Use the WCS information to add the pixel coordinates of each source
        Adds an ``x`` and ``y`` column to the catalog of fake sources.
    `trimFakeCat`
        Trim the fake cat to about the size of the input image.
    `mkFakeGalsimGalaxies`
        Use Galsim to make fake double sersic galaxies for each set of galaxy parameters in the input file.
    `mkFakeStars`
        Use the PSF information from the calexp to make a fake star using the magnitude information from the
        input file.
    `cleanCat`
        Remove rows of the input fake catalog which have half light radius, of either the bulge or the disk,
        that are 0.
    `addFakeSources`
        Add the fake sources to the calexp.

    Notes
    -----
    The ``calexp`` with fake souces added to it is written out as the datatype ``calexp_fakes``.
    """

    _DefaultName = "processCcdWithFakes"
    ConfigClass = ProcessCcdWithFakesConfig

    def __init__(self, schema=None, **kwargs):
        """Initalize things! This should go above in the class docstring
        """

        super().__init__(**kwargs)

        if schema is None:
            schema = SourceTable.makeMinimalSchema()
        self.schema = schema
        self.makeSubtask("insertFakes")
        self.makeSubtask("calibrate")

    def runQuantum(self, butlerQC, inputRefs, outputRefs):
        inputs = butlerQC.get(inputRefs)
        detectorId = inputs["exposure"].getInfo().getDetector().getId()

        if 'idGenerator' not in inputs.keys():
            inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId)

        expWcs = inputs["exposure"].getWcs()
        tractId = None
        if not self.config.doApplyExternalGlobalSkyWcs and not self.config.doApplyExternalTractSkyWcs:
            if expWcs is None:
                self.log.info("No WCS for exposure %s so cannot insert fake sources.  Skipping detector.",
                              butlerQC.quantum.dataId)
                return None
            else:
                inputs["wcs"] = expWcs
        elif self.config.doApplyExternalGlobalSkyWcs:
            externalSkyWcsCatalog = inputs["externalSkyWcsGlobalCatalog"]
            row = externalSkyWcsCatalog.find(detectorId)
            if row is None:
                self.log.info("No %s external global sky WCS for exposure %s so cannot insert fake "
                              "sources.  Skipping detector.", self.config.externalSkyWcsName,
                              butlerQC.quantum.dataId)
                return None
            inputs["wcs"] = row.getWcs()
        elif self.config.doApplyExternalTractSkyWcs:
            externalSkyWcsCatalogList = inputs["externalSkyWcsTractCatalog"]
            if tractId is None:
                tractId = externalSkyWcsCatalogList[0].dataId["tract"]
            externalSkyWcsCatalog = None
            for externalSkyWcsCatalogRef in externalSkyWcsCatalogList:
                if externalSkyWcsCatalogRef.dataId["tract"] == tractId:
                    externalSkyWcsCatalog = externalSkyWcsCatalogRef.get()
                    break
            if externalSkyWcsCatalog is None:
                usedTract = externalSkyWcsCatalogList[-1].dataId["tract"]
                self.log.warn(
                    f"Warning, external SkyWcs for tract {tractId} not found. Using tract {usedTract} "
                    "instead.")
                externalSkyWcsCatalog = externalSkyWcsCatalogList[-1].get()
            row = externalSkyWcsCatalog.find(detectorId)
            if row is None:
                self.log.info("No %s external tract sky WCS for exposure %s so cannot insert fake "
                              "sources.  Skipping detector.", self.config.externalSkyWcsName,
                              butlerQC.quantum.dataId)
                return None
            inputs["wcs"] = row.getWcs()

        if not self.config.doApplyExternalGlobalPhotoCalib and not self.config.doApplyExternalTractPhotoCalib:
            inputs["photoCalib"] = inputs["exposure"].getPhotoCalib()
        elif self.config.doApplyExternalGlobalPhotoCalib:
            externalPhotoCalibCatalog = inputs["externalPhotoCalibGlobalCatalog"]
            row = externalPhotoCalibCatalog.find(detectorId)
            if row is None:
                self.log.info("No %s external global photoCalib for exposure %s so cannot insert fake "
                              "sources.  Skipping detector.", self.config.externalPhotoCalibName,
                              butlerQC.quantum.dataId)
                return None
            inputs["photoCalib"] = row.getPhotoCalib()
        elif self.config.doApplyExternalTractPhotoCalib:
            externalPhotoCalibCatalogList = inputs["externalPhotoCalibTractCatalog"]
            if tractId is None:
                tractId = externalPhotoCalibCatalogList[0].dataId["tract"]
            externalPhotoCalibCatalog = None
            for externalPhotoCalibCatalogRef in externalPhotoCalibCatalogList:
                if externalPhotoCalibCatalogRef.dataId["tract"] == tractId:
                    externalPhotoCalibCatalog = externalPhotoCalibCatalogRef.get()
                    break
            if externalPhotoCalibCatalog is None:
                usedTract = externalPhotoCalibCatalogList[-1].dataId["tract"]
                self.log.warn(
                    f"Warning, external PhotoCalib for tract {tractId} not found. Using tract {usedTract} "
                    "instead.")
                externalPhotoCalibCatalog = externalPhotoCalibCatalogList[-1].get()
            row = externalPhotoCalibCatalog.find(detectorId)
            if row is None:
                self.log.info("No %s external tract photoCalib for exposure %s so cannot insert fake "
                              "sources.  Skipping detector.", self.config.externalPhotoCalibName,
                              butlerQC.quantum.dataId)
                return None
            inputs["photoCalib"] = row.getPhotoCalib()

        outputs = self.run(**inputs)
        butlerQC.put(outputs, outputRefs)

    def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None,
            icSourceCat=None, sfdSourceCat=None, externalSkyWcsGlobalCatalog=None,
            externalSkyWcsTractCatalog=None, externalPhotoCalibGlobalCatalog=None,
            externalPhotoCalibTractCatalog=None, idGenerator=None):
        """Add fake sources to a calexp and then run detection, deblending and
        measurement.

        Parameters
        ----------
        fakeCats : `list` of `lsst.daf.butler.DeferredDatasetHandle`
            Set of tract level fake catalogs that potentially cover this
            detectorVisit.
        exposure : `lsst.afw.image.exposure.exposure.ExposureF`
            The exposure to add the fake sources to.
        skyMap : `lsst.skymap.SkyMap`
            SkyMap defining the tracts and patches the fakes are stored over.
        wcs : `lsst.afw.geom.SkyWcs`, optional
            WCS to use to add fake sources.
        photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`, optional
            Photometric calibration to be used to calibrate the fake sources.
        exposureIdInfo : `lsst.obs.base.ExposureIdInfo`, optional
            Object that carries ID information for this image/catalog.
            Deprecated in favor of ``idGenerator``.
        icSourceCat : `lsst.afw.table.SourceCatalog`, optional
            Catalog to take the information about which sources were used for
            calibration from.
        sfdSourceCat : `lsst.afw.table.SourceCatalog`, optional
            Catalog produced by singleFrameDriver, needed to copy some
            calibration flags from.
        externalSkyWcsGlobalCatalog : `lsst.afw.table.ExposureCatalog`, \
                optional
            Exposure catalog with external skyWcs to be applied per config.
        externalSkyWcsTractCatalog : `lsst.afw.table.ExposureCatalog`, optional
            Exposure catalog with external skyWcs to be applied per config.
        externalPhotoCalibGlobalCatalog : `lsst.afw.table.ExposureCatalog`, \
                optional
            Exposure catalog with external photoCalib to be applied per config
        externalPhotoCalibTractCatalog : `lsst.afw.table.ExposureCatalog`, \
                optional
            Exposure catalog with external photoCalib to be applied per config.
        idGenerator : `lsst.meas.base.IdGenerator`, optional
            Object that generates Source IDs and random seeds.

        Returns
        -------
        resultStruct : `lsst.pipe.base.struct.Struct`
            Result struct containing:

            - outputExposure: `lsst.afw.image.exposure.exposure.ExposureF`
            - outputCat: `lsst.afw.table.source.source.SourceCatalog`

        Notes
        -----
        Adds pixel coordinates for each source to the fakeCat and removes
        objects with bulge or disk half light radius = 0 (if ``config.cleanCat
        = True``). These columns are called ``x`` and ``y`` and are in pixels.

        Adds the ``Fake`` mask plane to the exposure which is then set by
        `addFakeSources` to mark where fake sources have been added. Uses the
        information in the ``fakeCat`` to make fake galaxies (using galsim) and
        fake stars, using the PSF models from the PSF information for the
        calexp. These are then added to the calexp and the calexp with fakes
        included returned.

        The galsim galaxies are made using a double sersic profile, one for the
        bulge and one for the disk, this is then convolved with the PSF at that
        point.
        """
        fakeCat = self.composeFakeCat(fakeCats, skyMap)

        if wcs is None:
            wcs = exposure.getWcs()

        if photoCalib is None:
            photoCalib = exposure.getPhotoCalib()

        if self.config.doMatchVisit:
            fakeCat = self.getVisitMatchedFakeCat(fakeCat, exposure)

        self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)

        # detect, deblend and measure sources
        if idGenerator is None:
            if exposureIdInfo is not None:
                idGenerator = IdGenerator._from_exposure_id_info(exposureIdInfo)
            else:
                idGenerator = IdGenerator()
        returnedStruct = self.calibrate.run(exposure, idGenerator=idGenerator)
        sourceCat = returnedStruct.sourceCat

        sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)

        resultStruct = pipeBase.Struct(outputExposure=exposure, outputCat=sourceCat)
        return resultStruct

    def composeFakeCat(self, fakeCats, skyMap):
        """Concatenate the fakeCats from tracts that may cover the exposure.

        Parameters
        ----------
        fakeCats : `list` of `lsst.daf.butler.DeferredDatasetHandle`
            Set of fake cats to concatenate.
        skyMap : `lsst.skymap.SkyMap`
            SkyMap defining the geometry of the tracts and patches.

        Returns
        -------
        combinedFakeCat : `pandas.DataFrame`
            All fakes that cover the inner polygon of the tracts in this
            quantum.
        """
        if len(fakeCats) == 1:
            return fakeCats[0].get()
        outputCat = []
        for fakeCatRef in fakeCats:
            cat = fakeCatRef.get()
            tractId = fakeCatRef.dataId["tract"]
            # Make sure all data is within the inner part of the tract.
            outputCat.append(cat[
                skyMap.findTractIdArray(cat[self.config.insertFakes.ra_col],
                                        cat[self.config.insertFakes.dec_col],
                                        degrees=False)
                == tractId])

        return pd.concat(outputCat)

    def getVisitMatchedFakeCat(self, fakeCat, exposure):
        """Trim the fakeCat to select particular visit

        Parameters
        ----------
        fakeCat : `pandas.core.frame.DataFrame`
                    The catalog of fake sources to add to the exposure
        exposure : `lsst.afw.image.exposure.exposure.ExposureF`
                    The exposure to add the fake sources to

        Returns
        -------
        movingFakeCat : `pandas.DataFrame`
            All fakes that belong to the visit
        """
        selected = exposure.getInfo().getVisitInfo().getId() == fakeCat["visit"]

        return fakeCat[selected]

    def copyCalibrationFields(self, calibCat, sourceCat, fieldsToCopy):
        """Match sources in calibCat and sourceCat and copy the specified fields

        Parameters
        ----------
        calibCat : `lsst.afw.table.SourceCatalog`
            Catalog from which to copy fields.
        sourceCat : `lsst.afw.table.SourceCatalog`
            Catalog to which to copy fields.
        fieldsToCopy : `lsst.pex.config.listField.List`
            Fields to copy from calibCat to SoourceCat.

        Returns
        -------
        newCat : `lsst.afw.table.SourceCatalog`
            Catalog which includes the copied fields.

        The fields copied are those specified by `fieldsToCopy` that actually exist
        in the schema of `calibCat`.

        This version was based on and adapted from the one in calibrateTask.
        """

        # Make a new SourceCatalog with the data from sourceCat so that we can add the new columns to it
        sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema)
        sourceSchemaMapper.addMinimalSchema(sourceCat.schema, True)

        calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema, sourceCat.schema)

        # Add the desired columns from the option fieldsToCopy
        missingFieldNames = []
        for fieldName in fieldsToCopy:
            if fieldName in calibCat.schema:
                schemaItem = calibCat.schema.find(fieldName)
                calibSchemaMapper.editOutputSchema().addField(schemaItem.getField())
                schema = calibSchemaMapper.editOutputSchema()
                calibSchemaMapper.addMapping(schemaItem.getKey(), schema.find(fieldName).getField())
            else:
                missingFieldNames.append(fieldName)
        if missingFieldNames:
            raise RuntimeError(f"calibCat is missing fields {missingFieldNames} specified in "
                               "fieldsToCopy")

        if "calib_detected" not in calibSchemaMapper.getOutputSchema():
            self.calibSourceKey = calibSchemaMapper.addOutputField(afwTable.Field["Flag"]("calib_detected",
                                                                   "Source was detected as an icSource"))
        else:
            self.calibSourceKey = None

        schema = calibSchemaMapper.getOutputSchema()
        newCat = afwTable.SourceCatalog(schema)
        newCat.reserve(len(sourceCat))
        newCat.extend(sourceCat, sourceSchemaMapper)

        # Set the aliases so it doesn't complain.
        for k, v in sourceCat.schema.getAliasMap().items():
            newCat.schema.getAliasMap().set(k, v)

        select = newCat["deblend_nChild"] == 0
        matches = afwTable.matchXy(newCat[select], calibCat, self.config.matchRadiusPix)
        # Check that no sourceCat sources are listed twice (we already know
        # that each match has a unique calibCat source ID, due to using
        # that ID as the key in bestMatches)
        numMatches = len(matches)
        numUniqueSources = len(set(m[1].getId() for m in matches))
        if numUniqueSources != numMatches:
            self.log.warning("%d calibCat sources matched only %d sourceCat sources", numMatches,
                             numUniqueSources)

        self.log.info("Copying flags from calibCat to sourceCat for %s sources", numMatches)

        # For each match: set the calibSourceKey flag and copy the desired
        # fields
        for src, calibSrc, d in matches:
            if self.calibSourceKey:
                src.setFlag(self.calibSourceKey, True)
            # src.assign copies the footprint from calibSrc, which we don't want
            # (DM-407)
            # so set calibSrc's footprint to src's footprint before src.assign,
            # then restore it
            calibSrcFootprint = calibSrc.getFootprint()
            try:
                calibSrc.setFootprint(src.getFootprint())
                src.assign(calibSrc, calibSchemaMapper)
            finally:
                calibSrc.setFootprint(calibSrcFootprint)

        return newCat

class ProcessCcdWithVariableFakesConnections(ProcessCcdWithFakesConnections):
    ccdVisitFakeMagnitudes = cT.Output(
        doc="Catalog of fakes with magnitudes scattered for this ccdVisit.",
        name="{fakesType}ccdVisitFakeMagnitudes",
        storageClass="DataFrame",
        dimensions=("instrument", "visit", "detector"),
    )

class ProcessCcdWithVariableFakesConfig(ProcessCcdWithFakesConfig,
                                        pipelineConnections=ProcessCcdWithVariableFakesConnections):
    scatterSize = pexConfig.RangeField(
        dtype=float,
        default=0.4,
        min=0,
        max=100,
        doc="Amount of scatter to add to the visit magnitude for variable "
            "sources."
    )

class ProcessCcdWithVariableFakesTask(ProcessCcdWithFakesTask):
    """As ProcessCcdWithFakes except add variablity to the fakes catalog
    magnitude in the observed band for this ccdVisit.

    Additionally, write out the modified magnitudes to the Butler.
    """

    _DefaultName = "processCcdWithVariableFakes"
    ConfigClass = ProcessCcdWithVariableFakesConfig

    def run(self, fakeCats, exposure, skyMap, wcs=None, photoCalib=None, exposureIdInfo=None,
            icSourceCat=None, sfdSourceCat=None, idGenerator=None):
        """Add fake sources to a calexp and then run detection, deblending and
        measurement.

        Parameters
        ----------
        fakeCat : `pandas.core.frame.DataFrame`
            The catalog of fake sources to add to the exposure.
        exposure : `lsst.afw.image.exposure.exposure.ExposureF`
            The exposure to add the fake sources to.
        skyMap : `lsst.skymap.SkyMap`
            SkyMap defining the tracts and patches the fakes are stored over.
        wcs : `lsst.afw.geom.SkyWcs`, optional
            WCS to use to add fake sources.
        photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`, optional
            Photometric calibration to be used to calibrate the fake sources.
        exposureIdInfo : `lsst.obs.base.ExposureIdInfo`, optional
            Object that carries ID information for this image/catalog.
            Deprecated in favor of ``idGenerator``.
        icSourceCat : `lsst.afw.table.SourceCatalog`, optional
            Catalog to take the information about which sources were used for
            calibration from.
        sfdSourceCat : `lsst.afw.table.SourceCatalog`, optional
            Catalog produced by singleFrameDriver, needed to copy some
            calibration flags from.
        idGenerator : `lsst.meas.base.IdGenerator`, optional
            Object that generates Source IDs and random seeds.

        Returns
        -------
        resultStruct : `lsst.pipe.base.struct.Struct`
            Results struct containing:

            - outputExposure : Exposure with added fakes
              (`lsst.afw.image.exposure.exposure.ExposureF`)
            - outputCat : Catalog with detected fakes
              (`lsst.afw.table.source.source.SourceCatalog`)
            - ccdVisitFakeMagnitudes : Magnitudes that these fakes were
              inserted with after being scattered (`pandas.DataFrame`)

        Notes
        -----
        Adds pixel coordinates for each source to the fakeCat and removes
        objects with bulge or disk half light radius = 0 (if ``config.cleanCat
        = True``). These columns are called ``x`` and ``y`` and are in pixels.

        Adds the ``Fake`` mask plane to the exposure which is then set by
        `addFakeSources` to mark where fake sources have been added. Uses the
        information in the ``fakeCat`` to make fake galaxies (using galsim) and
        fake stars, using the PSF models from the PSF information for the
        calexp. These are then added to the calexp and the calexp with fakes
        included returned.

        The galsim galaxies are made using a double sersic profile, one for the
        bulge and one for the disk, this is then convolved with the PSF at that
        point.

        """
        fakeCat = self.composeFakeCat(fakeCats, skyMap)

        if wcs is None:
            wcs = exposure.getWcs()

        if photoCalib is None:
            photoCalib = exposure.getPhotoCalib()

        if idGenerator is None:
            if exposureIdInfo is not None:
                idGenerator = IdGenerator._from_exposure_id_info(exposureIdInfo)
            else:
                idGenerator = IdGenerator()

        band = exposure.getFilter().bandLabel
        ccdVisitMagnitudes = self.addVariability(
            fakeCat,
            band,
            exposure,
            photoCalib,
            idGenerator.catalog_id,
        )

        self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)

        # detect, deblend and measure sources
        returnedStruct = self.calibrate.run(exposure, idGenerator=idGenerator)
        sourceCat = returnedStruct.sourceCat

        sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat, self.config.srcFieldsToCopy)

        resultStruct = pipeBase.Struct(outputExposure=exposure,
                                       outputCat=sourceCat,
                                       ccdVisitFakeMagnitudes=ccdVisitMagnitudes)
        return resultStruct

    def addVariability(self, fakeCat, band, exposure, photoCalib, rngSeed):
        """Add scatter to the fake catalog visit magnitudes.

        Currently just adds a simple Gaussian scatter around the static fake
        magnitude. This function could be modified to return any number of
        fake variability.

        Parameters
        ----------
        fakeCat : `pandas.DataFrame`
            Catalog of fakes to modify magnitudes of.
        band : `str`
            Current observing band to modify.
        exposure : `lsst.afw.image.ExposureF`
            Exposure fakes will be added to.
        photoCalib : `lsst.afw.image.PhotoCalib`
            Photometric calibration object of ``exposure``.
        rngSeed : `int`
            Random number generator seed.

        Returns
        -------
        dataFrame : `pandas.DataFrame`
            DataFrame containing the values of the magnitudes to that will
            be inserted into this ccdVisit.
        """
        rng = np.random.default_rng(rngSeed)
        magScatter = rng.normal(loc=0,
                                scale=self.config.scatterSize,
                                size=len(fakeCat))
        visitMagnitudes = fakeCat[self.insertFakes.config.mag_col % band] + magScatter
        fakeCat.loc[:, self.insertFakes.config.mag_col % band] = visitMagnitudes
        return pd.DataFrame(data={"variableMag": visitMagnitudes})


#Lets get max number from a list

invoices = [6, 10, 20, 47, 69, 11, 50, 230, 20, 24, 39, 29]

count = 0

for number in invoices:
    if count < number:
        count = number

print(invoices)
print("Max = "+ str(count))
print("")
print('_____________OR_____________')
print("")

def min_num(list):
    smallest = list[0]
    for number in list:
        if smallest > number:
            smallest = number
    return smallest

def max_num(list):
    biggest = list[0]
    for number in list:
        if biggest < number:
            biggest = number
    return biggest

mini = min_num(invoices)
maxim = max_num(invoices)

print(invoices)
print("Minimum: "+ str(mini))
print("Maximum: "+ str(maxim))

Profit With Python/Module 1 - The Fundamentals of Python/MinMaxNum.py.py


from flask import render_template, request, redirect, flash, url_for
from app import app, db, models, mail
from flask_mail import Message

@app.route('/')
@app.route('/index')
def index():
    return render_template('index.html',
                           title='Welcome')

@app.route('/chapOnly')
def cFom():
    return render_template('conly.html',
                           title='Welcome')

@app.route('/list')
def list():
    users = models.User.query.all()
    return render_template('list.html',
                           title='List',
                           users=users)

@app.route('/chaplist')
def chaplist():
    users = models.Chap.query.all();
    return render_template('chaperones.html',
                           title='List',
                           users=users)

@app.route('/register', methods=['POST'])
def login():
    error = None
    n=1
    chapText = ""
    UserList=[]
    chapList=[]
    if request.method == 'POST':
        while(request.form['name'+str(n)]!=''):
            print(n)
            print(request.form['name'+str(n)])
            if(models.User.query.filter(models.User.name == request.form['name'+str(n)], models.User.parent == request.form['parent']).count()==0):
                UserList.append(models.User(name=request.form['name'+str(n)], age = int(request.form['age'+str(n)]), parent = request.form['parent'], parent_email = request.form['parent_email'], lunch = (request.form['lunch']!="false")
                , chaperone = (request.form['chap']!="false"), chapName = request.form['chapn'], notes = request.form['notes'+str(n)]))
                db.session.add(UserList[n-1])
                chapText+=request.form['name'+str(n)]+" Age:"+request.form['age'+str(n)]+", "
            else:
                return 'Error, '+request.form['name'+str(n)]+' is already registered.  Please email Shivam at smevawala@lgsinnovations.com for assistance'
            n+=1
        if(request.form['chap']!="false"):
            if(models.Chap.query.filter(models.Chap.name == request.form['chapn'], models.Chap.parent == request.form['parent']).count()==0):
                chap= models.Chap(name  = request.form['chapn'], parent = request.form['parent'], parent_email = request.form['parent_email'], kids = chapText)
                db.session.add(chap)
                chapList.append(chap)
            else:
                return 'Error, '+request.form['chapn']+' is already registered as a chaperone.  Please email Shivam at smevawala@lgsinnovations.com for assistance'

        db.session.commit()
        flash('You registered')
        msg=Message("Thanks for registering for LGS's Bring Your Child to Work Day!", sender="smevawala@lgsinnovations.com", recipients=[UserList[0].parent_email])
        #mail.send(msg)
        return render_template('thanks.html',
                               title='Thank You For Registering',
                               users=UserList, chaps=chapList)
    return redirect(url_for('index'))

@app.route('/registerC', methods=['POST'])
def regC():
    error = None
    UserList=[]
    chapList=[]
    if request.method == 'POST':
        if(models.Chap.query.filter(models.Chap.name == request.form['chapn'], models.Chap.parent == request.form['chapn']).count()==0):
            print("got here")
            chap= models.Chap(name  = request.form['chapn'], parent = request.form['chapn'], parent_email = "" ,kids = '')
            db.session.add(chap)
            chapList.append(chap)
            db.session.commit()
            flash('You registered')
            return render_template('thanks.html',
                                   title='Thank You For Registering',
                                   users=UserList, chaps=chapList)
        else:
            print("got here else")
            return 'Error, '+request.form['chapn']+' is already registered as a chaperone.  Please email Shivam at smevawala@lgsinnovations.com for assistance'
    return redirect(url_for('index'))


from turtle import Turtle

FONT = ("Courier", 24, "normal")
class Scoreboard(Turtle):
    def __init__(self):
        super().__init__()
        self.level = 0
        self.difficulty = 0.1
        self.hideturtle()
        self.color("white")
        self.penup()
        self.goto(0,250)
        self.write(f"Score: {self.level}", align="center", font=("Courier", 24, "normal"))

    def increase_difficulty(self):
        self.difficulty *= 0.7

    def update_score(self):
        self.hideturtle()
        self.color("white")
        self.level += 1
        self.penup()
        self.goto(0,0)
        self.write(f"Score: {self.level}", align="center", font=("Courier", 24, "normal"))


import streamlit as st
def calculate_emi(p,n,r):
	emi=p*(r/100)*(1+(r/100)**n)/(((1+(r/100))**n)-1)
	st.write("EMI Calculated=",emi)

st.title("EMI Calculator")
principal=st.slider("Loan Amount",1000.0,1000000.0)
tenure=st.slider("Time Period of Loan",1,50)
roi=st.slider("Rate of Interest",1,20)
if st.button("Calculate"):
	calculate_emi(principal,tenure,roi)


def isValid(s):
    stack = []
    dict = {"]":"[", "}":"{", ")":"("}
    for char in s:
        if char in dict.values():
            stack.append(char)
        elif stack == [] or dict[char] != stack.pop():
            return False
    return stack == []

if __name__ == "__main__":
    s = "{([[{}]])}"
    print(isValid(s))


import pandas as pd
import numpy as np
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split

# Leitura da base de dados
data = pd.read_csv('base_de_dados.csv')

# Definição das variáveis explicativas (X) e da variável resposta (y)
X = data[['idade', 'sexo', 'renda', 'historico_compras']]
y = data['contratou_x']

# Divisão da base de dados em treino e teste
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)

# Treinamento do modelo de regressão logística
model = LogisticRegression()
model.fit(X_train, y_train)

# Aplicação do modelo aos clientes que não contrataram o produto X
X_nao_contrataram = data[data['contratou_x'] == 0][['idade', 'sexo', 'renda', 'historico_compras']]
probabilidade_contratacao = model.predict_proba(X_nao_contrataram)

# Análise das diferenças de médias
media_gastos_contratantes = data[data['contratou_x'] == 1]['gastos'].mean()
media_gastos_nao_contratantes = data[data['contratou_x'] == 0]['gastos'].mean()

diferenca_medias = media_gastos_contratantes - media_gastos_nao_contratantes

print("Probabilidade de contratação do produto X para os clientes que não o contrataram:", probabilidade_contratacao[:, 1])
print("Diferença de médias de gastos entre os clientes que contrataram e os que não contrataram o produto X:", diferenca_medias)


from flask import Flask
from flask_restx import Api, Resource, fields
from werkzeug.middleware.proxy_fix import ProxyFix

app = Flask(__name__)
app.config['RESTX_MASK_SWAGGER'] = False
app.wsgi_app = ProxyFix(app.wsgi_app)
api = Api(app, version="1.0.0", title="astroneer", description="An astroneer app", prefix='/astro/v1')

ns = api.namespace("astroneer", description="Astroneer operations")  # todo why have namespaces at all?

DATABASE = {'modules': [], 'resources': [], 'planets': []}

resource_model = api.model("Resource", {
    "name": fields.String(required=True, description="The name of the resource"),
    "found": fields.List(fields.String, required=True, description="Planets on which this resource is found")
})
resource_list = api.model("ResourceList", {
    "resources": fields.List(fields.Nested(resource_model, description="Resource"), description="Resources"),
})

module_model = api.model("Module", {
    "name": fields.String(required=True, description="The name of the module"),
    "resource_cost": fields.List(fields.String, description="Resource cost to craft the module in a printer")
})
module_list = api.model("ModuleList", {
    "modules": fields.List(fields.Nested(module_model, description="Module"), description="Modules"),
})

def abort_if_module(module, **kwargs):
    test = [x['name'] for x in DATABASE['modules'] if x['name'] == module]
    if 'not_exists' in kwargs:
        test = not test
        api.abort(404, "module {} doesn't exist".format(module))
    if test:  # if empty, 404
        api.abort(400, "Module {} already exists".format(module))

def abort_if_resource(resource, **kwargs):
    test = [x['name'] for x in DATABASE['resources'] if x['name'] == resource]
    if 'not_exists' in kwargs:
        test = not test
        api.abort(404, "resource {} doesn't exist".format(resource))
    if test:  # if empty, 404
        api.abort(400, "Resource {} already exists".format(resource))

@ns.route("/")
class Debug(Resource):

    def get(self):
        """Debug print"""
        return DATABASE

resource_parser = api.parser()
resource_parser.add_argument("name", type=str, required=True, help="Resource name", location="form")
resource_parser.add_argument("found", type=str, required=True, help="Planets on which this resource is found",
                             location="form")
module_parser = api.parser()
module_parser.add_argument("name", type=str, required=True, help="Resource name", location="form")
module_parser.add_argument("resource_cost", type=str, required=True, help="Resource cost to print the module",
                           location="form")

#
# resources
#

@ns.route("/resource/<string:name_id>")
@api.doc(responses={404: "Resource not found"}, params={"name_id": "The resource name"})
class ResourceApi(Resource):
    """Show a single todo item and lets you delete them"""

    @api.doc(description="Resource name should be in the database")
    @api.marshal_with(resource_model)
    def get(self, name_id):
        """Fetch a given resource"""
        abort_if_resource(name_id, not_exists=True)
        return [x for x in DATABASE['resources'] if x['name'] == name_id][0]

    @api.doc(responses={204: "Resource deleted"})
    def delete(self, name_id):
        """Delete a given resource"""

        abort_if_resource(name_id, not_exists=True)
        DATABASE['resources'].remove([x for x in DATABASE['resources'] if x['name'] == name_id][0])
        return "", 204

    @api.doc(parser=resource_parser)
    @api.marshal_with(resource_model)
    def put(self, name_id):
        """Update a given resource"""
        abort_if_resource(name_id, not_exists=True)
        args = resource_parser.parse_args()
        resource = {
            "name": args["name"],
            "found": [x.strip() for x in args["found"].split(',')]
        }
        DATABASE['resources'].remove([x for x in DATABASE['resources'] if x['name'] == name_id][0])
        DATABASE['resources'].append(resource)
        return resource

@ns.route("/resource/")
class ResourceListApi(Resource):
    """Shows a list of all resources, and lets you POST to add new resources"""

    @api.marshal_list_with(resource_list)
    def get(self):
        """List all resources"""
        return {'resources': DATABASE['resources']}

    @api.doc(parser=resource_parser, responses={400: "Resource already exists"})
    @api.marshal_with(resource_model, code=201)
    def post(self):
        """Create a resource"""
        args = resource_parser.parse_args()
        abort_if_resource(args["name"], exists=True)
        resource = {
            "name": args["name"],
            "found": [x.strip() for x in args["found"].split(',')]
        }
        DATABASE['resources'].append(resource)
        return resource, 201

#
# modules
#

@ns.route("/module/<string:name_id>")
@api.doc(responses={404: "Module not found"}, params={"name_id": "The module name"})
class ModuleApi(Resource):
    """Show a single todo item and lets you delete them"""

    @api.doc(description="Module name should be in the database")
    @api.marshal_with(module_model)
    def get(self, name_id):
        """Fetch a given module"""
        abort_if_module(name_id, not_exists=True)
        return [x for x in DATABASE['modules'] if x['name'] == name_id][0]

    @api.doc(responses={204: "Module deleted"})
    def delete(self, name_id):
        """Delete a given module"""

        abort_if_module(name_id, not_exists=True)
        DATABASE['modules'].remove([x for x in DATABASE['modules'] if x['name'] == name_id][0])
        return "", 204

    @api.doc(parser=module_parser)
    @api.marshal_with(module_model)
    def put(self, name_id):
        """Update a given module"""

        abort_if_module(name_id, not_exists=True)
        args = module_parser.parse_args()
        module = {
            "name": args["name"],
            "found": [x.strip() for x in args["found"].split(',')]
        }
        DATABASE['modules'].remove([x for x in DATABASE['modules'] if x['name'] == name_id][0])
        DATABASE['modules'].append(module)
        return module

@ns.route("/module/")
class ModuleListApi(Resource):
    """Shows a list of all modules, and lets you POST to add new modules"""

    @api.marshal_list_with(module_list)
    def get(self):
        """List all modules"""
        return {'modules': DATABASE['modules']}

    @api.doc(parser=module_parser, responses={400: "Module already exists"})
    @api.marshal_with(module_model, code=201)
    def post(self):
        """Create a module"""
        args = module_parser.parse_args()
        abort_if_module(args["name"], exists=True)
        module = {
            "name": args["name"],
            "resource_cost": [x.strip() for x in args["resource_cost"].split(',')]
        }
        DATABASE['modules'].append(module)
        return module, 201

if __name__ == "__main__":
    app.run(debug=True)


from lxml.cssselect import CSSSelector
from lxml.html import fromstring
from lxml.etree import tostring
from urllib.request import urlopen
import os
from tqdm import tqdm

def getURLs(url):
	#otwiera stronę
	h = fromstring(urlopen(url+"testy.html").read())

	dict={}
	#pobiera komórki tabel (#oligonukleotydów, długość oligonukleotydu, #błędów)
	sel = CSSSelector('tr td')
	for e in sel(h):
		if e.sourceline not in dict:
			dict[e.sourceline]=[]
		if e.text is not None:
			dict[e.sourceline].append(int(e.text))#.decode('iso-8859-2')

	#pobiera url zbiorów testowych
	sel = CSSSelector('tr td a')
	for e in sel(h):
		dict[e.sourceline].append(url+e.get('href'))#.decode('iso-8859-2')

	#pobiera typy zbiorów
	# dict2={}
	# sel = CSSSelector('h4')
	# for e in sel(h):
		# if e.sourceline not in dict:
			# dict2[e.sourceline]=[]
		# if e.text is not None:
			# dict2[e.sourceline].append(e.text)#.decode('iso-8859-2')

	#dodaje typy zbiorów
	# for x in dict2:
		# for y in dict:
			# if len(dict[y])==4 and x<y:
				# dict[y].append(dict2[x][0])
				# dict[y].append(x)
			# elif len(dict[y])==6 and x>dict[y][5] and x<y:
				# dict[y][4]=dict2[x][0]
				# dict[y][5]=x

	#usuwa nadmiarowe dane
	# for x in dict:
		# dict[x]=dict[x][:-1]

	return dict;

def doFile(dir,nazwa,tab):
	if not os.path.exists(dir):
		os.makedirs(dir)
	f = open(dir+"/"+nazwa,'w')
	f.write(str(tab[0]) + " " + str(tab[1]) + " " + "nie pamiętam co tu powinno być\n")
	for line in urlopen(tab[3]).readlines():
		f.write(line.decode('iso-8859-2'))
	f.close()

if __name__=="__main__":
	dict = getURLs("http://www.cs.put.poznan.pl/mkasprzak/bio/");
	for i,d in enumerate(tqdm(dict)):
		doFile("datasets","plik"+str(i)+".txt",dict[d])


from django.contrib.auth.decorators import login_required
from django.urls import path, include
from django.views.generic import TemplateView

from travelling_companion import views

app_name = 'travelling_companion'
urlpatterns = [
    path('', views.TripListView.as_view(), name='home'),

    path('trip/<int:pk>', views.TripDetailView.as_view(), name='trip_detail'),
    path('trip/delete/<int:pk>', views.TripDeleteView.as_view(), name='trip_delete'),
    path('trip/create', views.TripCreateView.as_view(), name='trip_create'),
    path('trip/update/<int:pk>', views.TripUpdateView.as_view(), name='trip_update'),

    path('trip/<int:trip_id>/cost', views.CostView.as_view(), name='cost_form'),
    path('trip/<int:trip_id>/cost/<int:pk>', views.CostEditView.as_view(), name='cost_edit_form'),
    path('trip/cost/delete/<int:pk>', views.CostDelete.as_view(), name='cost_delete'),

    path('trip/<int:trip_id>/flight', views.FlightView.as_view(), name='flight_form'),
    path('trip/<int:trip_id>/flight/<int:pk>', views.FlightEditView.as_view(), name='flight_edit_form'),
    path('trip/flight/delete/<int:pk>', views.FlightDelete.as_view(), name='flight_delete'),

    path('trip/<int:trip_id>/accommodation', views.AccommodationView.as_view(), name='accommodation_form'),
    path('trip/<int:trip_id>/accommodation/<int:pk>', views.AccommodationEditView.as_view(), name='accommodation_edit'),
    path('trip/accommodation/delete/<int:pk>', views.AccommodationDelete.as_view(), name='accommodation_delete'),

    path('trip/accept/<int:pk>', views.AccepteTrip.as_view(), name='trip_accept'),
    path('trip/approve/<int:pk>', views.ApproveTrip.as_view(), name='trip_approve'),
    path('trip/<int:pk>/add_person/<int:person_id>', views.AddPersonTrip.as_view(), name='trip_add_person'),
    path('trip/remove_person/<int:pk>', views.RemovePersonTrip.as_view(), name='trip_remove_person'),

    path('location/', views.LocationView.as_view(), name='location_form'),
    path('location/<int:pk>', views.LocationEditView.as_view(), name='location_edit_form'),
    path('location/delete/<int:pk>', views.LocationDelete.as_view(), name='location_delete'),

    path('person/', views.PersonListView.as_view(), name='person_list'),
    path('person/delete/<int:pk>', views.PersonDeleteView.as_view(), name='person_delete'),
    path('person/create', views.PersonCreateView.as_view(), name='person_create'),
    path('person/update/<int:pk>', views.PersonUpdateView.as_view(), name='person_update'),

    path('passport/create', views.PassportCreateView.as_view(), name='passport_create'),

]