from  selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
import time
import requests
import random
import sys
import json

options = webdriver.ChromeOptions()

# setting profile
options.user_data_dir = "c:\\temp\\profile"

# another way to set profile is the below (which takes precedence if both variants are used
options.add_argument('--no-sandbox')
options.add_argument('--user-data-dir=c:\\temp\\testBrowser')
#options.add_argument('--incognito')
#options.add_argument('--start-fullscreen')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('disable-infobars')
options.add_argument('--disable-gpu')
options.add_argument('--disable-blink-features=AutomationControlled')
#options.add_argument('--start-maximized')
options.add_experimental_option("excludeSwitches", ["enable-automation","enable-logging"])
options.add_experimental_option('useAutomationExtension', False)
# just some options passing in to skip annoying popups
options.add_argument('--no-first-run --no-service-autorun --password-store=basic')
if __name__ == "__main__":
    driver = webdriver.Chrome(options=options)
    driver.set_window_size(800, 600)
    #driver.minimize_window()
    action = ActionChains(driver)

    with driver:
        #buka halaman situsproperti
        driver.get('https://www.situsproperti.com/properti-dijual')
        #clik wa
        nomor = []
        Kontak = []
        elem = "//a[contains(@href,\'whatsapp.com')]"
        try:
            WebDriverWait(driver,10).until(EC.presence_of_all_elements_located((By.XPATH,elem)))
        except:
            print('kesalahan webdriver shortcut whatsapp')
        else:
            try:
                whatsapp = driver.find_elements(By.XPATH,elem)
                #read all numbers
                for obj in whatsapp:
                    link = obj.get_attribute('href')
                    phone = link.split('=+')[1]
                    phone = phone.split('&text')[0]
                    nomor.append(phone)
                    kontak = set(nomor)

            except:
                print('nampaknya nomor whatsapp tidak ada')
        print(kontak)

        time.sleep(10)


#Standard Tic tac toe game
#Author: Landon Carter

def main():
    board = create_board()
    player = assign_player('')

    while not (if_win(board) or tie(board)):
        show_board(board)
        players_turn(player, board)
        player = assign_player(player)
    show_board(board)
    print('Good game. Thanks for playing!')

    if if_win(board) == True:
        player = assign_player(player)
        print(f'{player} is the winner')
    else:
        print("It's a Tie!")

#Create board
def create_board():
    board = []
    for i in range(9):
        board.append(i+1)
    return board

#Show board
def show_board(board):
    print(f'{board[0]}|{board[1]}|{board[2]}')
    print('-+-+-')
    print(f'{board[3]}|{board[4]}|{board[5]}')
    print('-+-+-')
    print(f'{board[6]}|{board[7]}|{board[8]}')

#assign player
def assign_player(new_player):
    if new_player == '' or new_player == 'o':
        return 'x'
    elif new_player == 'x':
        return 'o'

#players Turn
def players_turn(player, board):
        turn = int(input(f"{player}'s turn to choose a square(1-9): "))
        board[turn - 1] = player

#if_Win
def if_win(board):
    while not (board[0] == board[1] == board[2] or
            board[3] == board[4] == board[5] or
            board[6] == board[7] == board[8] or
            board[0] == board[3] == board[6] or
            board[1] == board[4] == board[7] or
            board[2] == board[5] == board[8] or
            board[0] == board[4] == board[8] or
            board[2] == board[4] == board[6]):
            return False
    return True

#Tie
def tie(board):
    for i in range(9):
        if board[i] != 'x' and board[i] != 'o':
            return(False)
    return True

if __name__ == "__main__":
    main()


import os
import pandas as pd
import requests
import csv

session = requests.session()

headers = {
    'user-agent': "Chrome/87.0.4280.88"
}

def makeDataset(url):
    with open("dataset.csv", "w") as f:
        f.write(session.get(url).text)

    with open("dataset.csv", "r") as f:
        dataset = csv.reader(f)
        niftyData = []
        stockData = []

        for idx, row in enumerate(dataset):
            if 8 <= idx <= 67:
                niftyData.append(row)
            if 120 <= idx:
                stockData.append(row)
    os.remove("dataset.csv")
    return pd.DataFrame(niftyData), pd.DataFrame(stockData)

def getTodayData() -> object:
    webData = session.get(url="https://www.nseindia.com/api/merged-daily-reports?key=favCapital", headers=headers)
    return makeDataset(webData.json()[1]['link'])

# use it likes this
# nifty_data, companies_data = getTodayData()


from sklearn.ensemble import RandomForestClassifier
from sklearn.mixture import GMM
from xgboost import XGBClassifier

def classifiers(model):
    clfs = {
        'random_forest': {
            'name': 'sklearn-random_forest_classifier',
            'model':  RandomForestClassifier,
            'parameters': {}},
        'gradient_boosting': {
            'name': 'xgboost-xgb_classifier',
            'model': XGBClassifier,
            'parameters': {}},
        'gmm': {
            'name': 'sklearn-gmm',
            'model': GMM,
            'parameters': {}}}

    if model == 'all':
        return clfs.values()

    return [clfs[model]]


import os

from charmhelpers.core.hookenv import (
    status_set,
    config,
)

from charms.reactive import (
    when,
    when_not,
    hook,
    is_state,
    set_state,
    remove_state,
)

from charmhelpers.fetch import (
    apt_install,
)

def install_cuda():
    pass

def install_tensorflow(version, cuda=False):
    import pip

    if version == 'stable':
        # REFACTOR don't do this
        version = '0.5.0'

    status_set('maintenance', 'installing %s' % version)
    url = 'https://storage.googleapis.com/tensorflow/linux/{type}/tensorflow-{version}-cp27-none-linux_x86_64.whl'
    pip.main(['install',
              url.format(version=version, type='cpu')])
    status_set('maintenance', 'installed')

def install_tensorflow_source():
    pass

def install_deps(source, cuda):
    if is_state('tensorflow.deps-installed'):
        return
    # MAGIC
    if source:
        apt_install(['python-numpy',  'swig', 'python-dev'])

    if cuda:
        install_cuda()

    apt_install(['python-pip', 'python-dev'])
    set_state('tensorflow.deps-installed')

@hook('config-changed')
def configure():
    cfg = config()
    cuda = cfg.get('tensorflow-cuda')
    version = cfg.get('tensorflow-version')

    if version == 'source':
        status_set('blocked', 'Can not install tensorflow from source yet.')
        return

    if cuda:
        status_set('blocked', 'Please dont use me')

    if cfg.changed('tensorflow-cuda') or cfg.changed('version') and is_state('tensorflow.deps-installed'):
        remove_state('tensorflow.deps-installed')

    install_deps(version == 'source', cuda)

    if version == 'source':
        install_tensorflow_source()
    else:
        install_tensorflow(version, cuda)

    set_state('tensorflow.ready')

@hook('upgrade-charm')
def workaround():
    remove_state('tensorflow.deps-installed')


from textblob import TextBlob

def modify_user_data(user_d_list):
    """
    Add url domain field to each tweet in each user data object.
    Url domain field contains list of domains corresponding to list of urls.
    """
    for user in user_d_list:
        for tweet in user['tweets']:
            blob = TextBlob(tweet['text'])
            tweet['sentiment'] = blob.sentiment.polarity
    return


import matplotlib.pyplot as plt
import numpy as np

in_time = [(0,27), (1,25), (2,16), (3,19), (4,26), (5,20), (6,19), (7,17), (8,10), (9,5), (10,4), (11,4), (12,2)]
cls_late = [(5,3), (6,5), (7,8), (8,15), (9,17), (10,18), (11,19), (12,16), (13,9), (14,8), (15,8)]

X,Y = zip(*in_time)
X2,Y2 = zip(*cls_late)

bar_width = 0.9

plt.bar(X,Y,bar_width,color="blue",alpha=0.75,label="Dung gio")

bar_width = 0.8

plt.bar(X2,Y2,bar_width,color="red",alpha=0.75,label="Tre")

plt.legend(loc='upper right')

plt.show()

in_time_dict = dict(in_time)
too_late_dict = dict(cls_late)

def catch_the_train(min):
    s = in_time_dict.get(min,0)
    if s == 0:
        return 0
    else:
        m = too_late_dict.get(min,0)
        return s/(s+m)
for minutes in range(0,15):
    print(minutes,catch_the_train(minutes))


import io
import logging
import requests
from pastehunter.common import parse_config
conf = parse_config()

logger = logging.getLogger('pastehunter')

def upload_file(raw_file, paste_object):
    try:
        task_id = send_to_cuckoo(raw_file, paste_object["pasteid"])
        paste_object["Cuckoo Task ID"] = task_id
        logger.info("exe submitted to Cuckoo with task id {0}".format(task_id))
    except Exception as e:
        logger.error("Unabled to submit sample to cuckoo")

    # Send any updated json back
    return paste_object

def send_to_cuckoo(raw_exe, pasteid):
    cuckoo_ip = conf["sandboxes"]["cuckoo"]["api_host"]
    cuckoo_port = conf["sandboxes"]["cuckoo"]["api_port"]
    cuckoo_host = 'http://{0}:{1}'.format(cuckoo_ip, cuckoo_port)
    submit_file_url = '{0}/tasks/create/file'.format(cuckoo_host)
    files = {'file': ('{0}.exe'.format(pasteid), io.BytesIO(raw_exe))}
    submit_file = requests.post(submit_file_url, files=files).json()
    task_id = None
    try:
        task_id = submit_file['task_id']
    except KeyError:
        try:
            task_id = submit_file['task_ids'][0]
        except KeyError:
            logger.error(submit_file)

    return task_id


from sqlalchemy.orm import Session
from fastapi import HTTPException, status, Response, Depends
from ..models import toppings as model
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy import func
from datetime import datetime
from ..models.toppings import Topping
from ..models.orders import Order
from ..models.pizzas import Pizza, pizza_topping_association

def read_all(db: Session):
    try:
        result = db.query(model.Topping).all()
    except SQLAlchemyError as e:
        error = str(e.__dict__['orig'])
        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=error)
    return result

def read_one(db: Session, item_id):
    try:
        item = db.query(model.Topping).filter(model.Topping.id == item_id).first()
        if not item:
            raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Id not found!")
    except SQLAlchemyError as e:
        error = str(e.__dict__['orig'])
        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=error)
    return item

def get_most_popular_topping(db: Session, start_date: datetime, end_date: datetime):
    # Join the necessary tables
    subquery = db.query(Topping.name, func.count(Topping.id).label('topping_count')).join(
        pizza_topping_association).join(Pizza).join(Order).filter(
        Order.order_date >= start_date, Order.order_date <= end_date).group_by(Topping.name).subquery()

    # Get the most popular topping by counting occurrences
    result = db.query(subquery.c.name).order_by(subquery.c.topping_count.desc()).first()
    return result


#!/usr/bin/env python3
import argparse
import serial
from time import sleep
import datetime

"""
COMMAND:
    `$ python3 sender.py COM??`
    ?? はデバイスマネージャで確認(USB Serial Portとして出てくる)

SETING:
    0. 'pip3 install pyserial'
    1. @PowerShell: Find out which port to connect. `usbipd wsl list`
    2. `usbipd wsl attach --distribution Ubuntu-20.04 --busid <BUSID>`
    3. Search connected USB list from`lsusb`
    4. `sudo chmod 777 /dev/ttyUSB0`
    Finaly: `usbipd wsl detach --busid <busid>`

USAGE:
    1. コライドンを手持ちの上から2番目に置いて6匹埋める
    2. ライド選択してライドフォルムに戻す
    3. ボックスを開いてボックス1に移動
    4. コマンドの実行 `python3 ./sv/item.py /dev/ttyUSB0 --loop 10`
"""

parser = argparse.ArgumentParser()
parser.add_argument('port')
parser.add_argument('-l', '--loop', type=int, default=1)
args = parser.parse_args()

def send(msg, duration=0):
    now = datetime.datetime.now()
    print(f'{now}: {msg}')
    ser.write(f'{msg}\r\n'.encode('utf-8'))
    sleep(duration)
    ser.write(b'RELEASE\r\n')

ser = serial.Serial(args.port, 9600)
sleep(2)
send('Button LCLICK', 0.1)

def main():
    count = 0

    try:
        for _ in range(0, args.loop):
            send('Button X', 0.2)
            sleep(0.2)
            send('Button X', 0.2)
            sleep(0.3)
            send('Button L', 0.2)
            sleep(0.3)

            # Select Koraidon
            send('Button A', 0.2)
            sleep(0.2)
            send('LY MIN', 0.2)
            sleep(0.2)
            send('LY MIN', 0.2)
            sleep(0.2)
            send('LY MIN', 0.2)
            sleep(0.2)
            send('Button A', 0.2)
            sleep(0.6)

            count += 1
            print("  counter: {}/{}".format(count, args.loop))

            # Close box
            send('Button B', 0.2)
            sleep(2.4)

            # Change to Ride form
            send('LX MIN', 0.2)
            sleep(0.4)
            send('Button A', 0.2)
            sleep(0.4)
            send('LY MIN', 0.2)
            sleep(0.2)
            send('LY MIN', 0.2)
            sleep(0.2)
            send('Button A', 0.2)
            sleep(0.9)
            send('Button A', 0.2)
            sleep(0.3)
            send('Button A', 0.2)
            sleep(2.7)
            send('Button A', 0.2)
            sleep(1.0)

            # Open Box
            send('LX MAX', 0.2)
            sleep(0.4)
            send('LY MAX', 0.2)
            sleep(0.2)
            send('LY MAX', 0.2)
            sleep(0.2)
            send('Button A', 0.2)
            sleep(3.0)

    except KeyboardInterrupt:
        print('Bye...')
        ser.close()

if __name__ == "__main__":
    main()


from collections import defaultdict

arr = [1, 3, 9, 9, 27, 81]
r = 3
dict1 = defaultdict(int)
dict2 = defaultdict(int)
count = 0
for k in arr:
    count += dict2[k]
    dict2[k * r] += dict1[k]
    dict1[k * r] += 1
print(count)


# using HTTPS or HTTP
HOST_PROTOCOL = "https://"

LOGIN_URL = '/backend/signin'

LOGIN_REDIRECT_URL = '/backend/signin'

LOGOUT_REDIRECT_URL = LOGIN_URL

USER_ACTIVATOR = 'aurora.pilar.activator'

SWEETIFY_SWEETALERT_LIBRARY = 'sweetalert2'

# enable or disable maintenance mode
MAINTENANCE = False

MAINTENANCE_MESSAGE = None


from tensorflow.keras.layers import Layer
import tensorflow as tf

class Model(Layer):

    def __init__(self):

        # 3 conv/pool blocks with 2 conv layers each. 2 dense hidden layers.

        # init super class
        super(Model, self).__init__()

        # define layers
        self.conv_layer_1 = tf.keras.layers.Conv2D(filters=32, kernel_size=3, padding='same', activation='relu', input_shape=(128,128,3))
        self.conv_layer_1_2 = tf.keras.layers.Conv2D(filters=32, kernel_size=3, padding='same', activation='relu')
        self.max_pool_layer_1 = tf.keras.layers.MaxPool2D()

        self.conv_layer_2 = tf.keras.layers.Conv2D(filters=32, kernel_size=3, padding='same', activation='relu')
        self.conv_layer_2_2 = tf.keras.layers.Conv2D(filters=32, kernel_size=3, padding='same', activation='relu')
        self.norm_layer_2 = tf.keras.layers.BatchNormalization()
        self.max_pool_layer_2 = tf.keras.layers.MaxPool2D()

        self.conv_layer_3 = tf.keras.layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu')
        self.conv_layer_3_2 = tf.keras.layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu')
        self.norm_layer_3 = tf.keras.layers.BatchNormalization()
        self.max_pool_layer_3 = tf.keras.layers.MaxPool2D()

        self.mlp_input = tf.keras.layers.GlobalAveragePooling2D()
        self.hidden_layer_1 = tf.keras.layers.Dense(units=128, activation=tf.keras.activations.sigmoid)
        self.hidden_layer_2 = tf.keras.layers.Dense(units=64, activation=tf.keras.activations.sigmoid)
        self.output_layer = tf.keras.layers.Dense(units=2, activation=tf.keras.activations.softmax)

    @tf.function
    def call(self, x):

        x = self.conv_layer_1(x)
        x = self.conv_layer_1_2(x)
        x = self.max_pool_layer_1(x)

        x = self.conv_layer_2(x)
        x = self.conv_layer_2_2(x)
        x = self.norm_layer_2(x)
        x = self.max_pool_layer_2(x)

        x = self.conv_layer_3(x)
        x = self.conv_layer_3_2(x)
        x = self.norm_layer_3(x)
        x = self.max_pool_layer_3(x)

        x = self.mlp_input(x)
        x = self.hidden_layer_1(x)
        x = self.hidden_layer_2(x)
        x = self.output_layer(x)

        return x


def bagproblem(w, wt, val, n):
	bag = [[0 for x in range(w + 1)] for x in range(n + 1)]

	for i in range(n + 1):
		for w in range(w + 1):
			if i == 0 or w == 0:
				bag[i][w] = 0
			elif wt[i-1] <= w:
				bag[i][w] = max(val[i-1] + bag[i-1][w-wt[i-1]], bag[i-1][w])
			else:
				bag[i][w] = bag[i-1][w]

	return bag[n][w]

w = 7
val = [1, 4, 6]
wt = [1, 6, 8]
n = len(val)

print(bagproblem(w, wt, val, n))


from flask import Flask
from flask_mysqldb import MySQL
from flask import Flask, render_template, redirect, url_for
mysql = MySQL()
app = Flask(__name__)
# My SQL Instance configurations
# Change the HOST IP and Password to match your instance configurations
app.config['MYSQL_USER'] = 'root'
app.config['MYSQL_PASSWORD'] = 'thomas'
app.config['MYSQL_DB'] = 'studentbook'
app.config['MYSQL_HOST'] = '35.197.205.126'
mysql.init_app(app)

# The first route to access the webservice from http://external-ip:5000
#@pp.route("/add") this will create a new endpoints that can be accessed using http://external-ip:5000/add
@app.route("/")
def view(): # Name of the method
    cur = mysql.connection.cursor() #create a connection to the SQL instance
    cur.execute('''SELECT * FROM students''') # execute an SQL statment
    rv = cur.fetchall() #Retreive all rows returend by the SQL statment
    return render_template("index.html", rv=rv)

@app.route("/add")
def add(): # Name of the method
    cur = mysql.connection.cursor() #create a connection to the SQL instance
    cur.execute('''INSERT INTO students (studentName, email) VALUES ("Thomas", "c15300756@mydit.ie")''') # execute an SQL statment
    mysql.connection.commit()
    return redirect(url_for("view"))

@app.route("/update")
def update(): # Name of the method
    cur = mysql.connection.cursor() #create a connection to the SQL instance
    cur.execute('''UPDATE students SET studentName="Thomas2" WHERE studentName="Thomas"''') # execute an SQL statment
    mysql.connection.commit()
    return redirect(url_for("view"))

@app.route("/delete")
def delete(): # Name of the method
    cur = mysql.connection.cursor() #create a connection to the SQL instance
    cur.execute('''DELETE FROM students WHERE studentName="Thomas"''') # execute an SQL statment
    mysql.connection.commit()
    return redirect(url_for("view"))

if __name__ == "__main__":
        app.run(host='0.0.0.0', port='5000') #Run the flask app at port 5000


from django.contrib import admin
from django.urls import path

from . import views

app_name = 'mainshop'
urlpatterns = [
    path('', views.main, name='index'),
    path('products/', views.products, name='products'),
    path('contact/', views.contact, name='contact'),
    path('admin/', admin.site.urls, name='admin'),
]


# myapp/urls.py
from django.urls import path
from .views import custom_login
from django.contrib import admin
from .views import qr_code_decoder, video_feed,event_list,add_to_purchased,stop_stream,land,gallery_view,export_view,remove_purchased, tour

urlpatterns = [
    path('admin/', admin.site.urls),
    path('qr_code_decoder/', qr_code_decoder, name='qr_code_decoder'),
    path('video_feed/', video_feed, name='video_feed'),
    path('remove_purchased/<int:event_id>', remove_purchased, name='remove_purchased'),
    path('', land,name="land"),
    path('custom_login/', custom_login, name='custom_login'),
    path('export_view/', export_view, name='export'),
    path('gallery/', gallery_view, name='gallery'),
    path('event_list/',event_list, name='event_list'),
    path('add_to_purchased/<int:event_id>/', add_to_purchased, name='add_to_purchased'),
    path('stop_stream/', stop_stream, name='stop_stream'),
    path('tour/', tour, name='tour'),
]


import glob
import os
import shutil
from functools import reduce
import re

def delete_files_in_dir(
    idp, ext=None, target_str_or_list=None, sort_result=True, recursive=False
):

    """ext can be a list of extensions or a single extension
    (e.g. ['.jpg', '.png'] or '.jpg')
    """

    fps = get_file_paths_in_dir(
        idp,
        ext=ext,
        target_str_or_list_in_fn=target_str_or_list,
        sort_result=sort_result,
        recursive=recursive,
    )

    for fp in fps:
        assert os.path.isfile(fp)
        os.remove(fp)

def glob_re(pattern, strings):
    # https://stackoverflow.com/questions/13031989/regular-expression-usage-in-glob-glob
    """Alternative to glob using regular expressions.

    Aims to overcome limitations of glob (e.g. no optional strings supported).
    Usage: filenames = glob_re(r'.*(abc|123|a1b).*', os.listdir())
    """
    return filter(re.compile(pattern).match, strings)

def natural_key(some_string):
    """See http://www.codinghorror.com/blog/archives/001018.html"""
    return [
        int(s) if s.isdigit() else s for s in re.split(r"(\d+)", some_string)
    ]

def check_ext(ext):
    # Check if extension is valid (contains a leading dot)
    if isinstance(ext, list):
        for ele in ext:
            assert ele[0] == ".", "Invalid extension, leading dot missing"
    else:
        assert ext[0] == ".", "Invalid extension, leading dot missing"

def get_regex_fps_in_dp(idp, search_regex, ignore_regex):
    assert os.path.isdir(idp)

    glob_search_expr = os.path.join(idp, search_regex)
    search_ifps = glob.glob(glob_search_expr, recursive=True)
    err_msg = f"Found no images using {glob_search_expr}"
    assert len(search_ifps), err_msg

    glob_ignore_expr = os.path.join(idp, ignore_regex)
    ignore_ifps = glob.glob(glob_ignore_expr, recursive=True)
    # Use a set to accelerate existence check: https://wiki.python.org/moin/TimeComplexity
    ignore_ifps_set = set(ignore_ifps)
    ifps = [ifp for ifp in search_ifps if ifp not in ignore_ifps_set]
    err_msg = (
        f"Found no images using {{{glob_search_expr} / {glob_ignore_expr}}}"
    )
    assert len(ifps), err_msg

    return ifps

def get_file_paths_in_dir(
    idp,
    ext=None,
    target_str_or_list_in_fn=None,
    target_str_or_list_in_fp=None,
    ignore_str_or_list_in_fn=None,
    ignore_str_or_list_in_fp=None,
    base_name_only=False,
    without_ext=False,
    sort_result=True,
    natural_sorting=False,
    recursive=False,
):

    """ext can be a list of extensions or a single extension
    (e.g. ['.jpg', '.png'] or '.jpg')
    """

    if recursive:
        ifp_s = []
        for root, dirs, files in os.walk(idp):
            ifp_s += [os.path.join(root, ele) for ele in files]
    else:
        ifp_s = [
            os.path.join(idp, ele)
            for ele in os.listdir(idp)
            if os.path.isfile(os.path.join(idp, ele))
        ]

    if ext is not None:
        if isinstance(ext, list):
            ext = [ele.lower() for ele in ext]
            check_ext(ext)
            ifp_s = [
                ifp for ifp in ifp_s if os.path.splitext(ifp)[1].lower() in ext
            ]
        else:
            ext = ext.lower()
            check_ext(ext)
            ifp_s = [
                ifp for ifp in ifp_s if os.path.splitext(ifp)[1].lower() == ext
            ]

    if target_str_or_list_in_fn is not None:
        if type(target_str_or_list_in_fn) == str:
            target_str_or_list_in_fn = [target_str_or_list_in_fn]
        for target_str in target_str_or_list_in_fn:
            ifp_s = [
                ifp for ifp in ifp_s if target_str in os.path.basename(ifp)
            ]

    if target_str_or_list_in_fp is not None:
        if type(target_str_or_list_in_fp) == str:
            target_str_or_list_in_fp = [target_str_or_list_in_fp]
        for target_str in target_str_or_list_in_fp:
            ifp_s = [ifp for ifp in ifp_s if target_str in ifp]

    if ignore_str_or_list_in_fn is not None:
        if type(ignore_str_or_list_in_fn) == str:
            ignore_str_or_list_in_fn = [ignore_str_or_list_in_fn]
        for ignore_str in ignore_str_or_list_in_fn:
            ifp_s = [
                ifp for ifp in ifp_s if ignore_str not in os.path.basename(ifp)
            ]

    if ignore_str_or_list_in_fp is not None:
        if type(ignore_str_or_list_in_fp) == str:
            ignore_str_or_list_in_fp = [ignore_str_or_list_in_fp]
        for ignore_str in ignore_str_or_list_in_fp:
            ifp_s = [ifp for ifp in ifp_s if ignore_str not in ifp]

    if base_name_only:
        ifp_s = [os.path.basename(ifp) for ifp in ifp_s]

    if without_ext:
        ifp_s = [os.path.splitext(ifp)[0] for ifp in ifp_s]

    if sort_result:
        if natural_sorting:
            ifp_s = sorted(ifp_s, key=natural_key)
        else:
            ifp_s = sorted(ifp_s)

    return ifp_s

def get_image_file_paths_in_dir(
    idp,
    base_name_only=False,
    without_ext=False,
    sort_result=True,
    recursive=True,
    target_str_or_list=None,
):
    return get_file_paths_in_dir(
        idp,
        ext=[".jpg", ".png"],
        target_str_or_list_in_fn=target_str_or_list,
        base_name_only=base_name_only,
        without_ext=without_ext,
        sort_result=sort_result,
        recursive=recursive,
    )

def get_corresponding_files_in_directories(
    idp_1,
    idp_2,
    ext_1=None,
    suffix_2="",
    get_correspondence_callback=None,
    sort_result=True,
):

    if get_correspondence_callback is None:

        def get_correspondence_callback(fn_1):
            return fn_1 + suffix_2

    potential_fn_1_list = get_file_paths_in_dir(
        idp_1,
        ext=ext_1,
        base_name_only=True,
        sort_result=sort_result,
    )
    fp_1_list = []
    fp_2_list = []
    for fn_1 in potential_fn_1_list:
        fp_2 = os.path.join(idp_2, get_correspondence_callback(fn_1))
        # print(fp_2)
        if os.path.isfile(fp_2):
            fp_1_list.append(os.path.join(idp_1, fn_1))
            fp_2_list.append(fp_2)
    return fp_1_list, fp_2_list

def delete_subdirs(idp, filter_dp, recursive=False, dry_run=True):

    sub_dirs = get_subdirs(idp, recursive=recursive)
    sub_dirs_to_delete = []
    for sub_dir in sub_dirs:
        if sub_dir.endswith(filter_dp):
            sub_dirs_to_delete.append(sub_dir)

    if dry_run:
        print("Dry run! (Files will not be deleted)")
    else:
        for dir in sub_dirs_to_delete:
            shutil.rmtree(dir, ignore_errors=True)
    print("sub_dirs_to_delete:")
    print(sub_dirs_to_delete)

def get_subdirs(idp, base_name_only=False, recursive=False):

    if recursive:
        sub_dps = []
        if base_name_only:
            for root, dirs, files in os.walk(idp):
                sub_dps += [name for name in dirs]
        else:
            for root, dirs, files in os.walk(idp):
                sub_dps += [os.path.join(root, sub_dn) for sub_dn in dirs]
    else:
        sub_dns = [
            name
            for name in os.listdir(idp)
            if os.path.isdir(os.path.join(idp, name))
        ]
        if base_name_only:
            sub_dps = sub_dns
        else:
            sub_dps = [os.path.join(idp, sub_dn) for sub_dn in sub_dns]

    return sub_dps

def get_stem(ifp, base_name_only=True):
    if base_name_only:
        ifp = os.path.basename(ifp)
    return os.path.splitext(ifp)[0]

def get_basename(ifp):
    return os.path.basename(ifp)

def mkdir_safely(odp):
    if not os.path.isdir(odp):
        os.mkdir(odp)

def makedirs_safely(odp):
    if not os.path.isdir(odp):
        os.makedirs(odp)

def ensure_trailing_slash(some_path):
    return os.path.join(some_path, "")

def get_first_valid_path(list_of_paths):
    first_valid_path = None
    for path in list_of_paths:
        if first_valid_path is None and (
            os.path.isdir(path) or os.path.isfile(path)
        ):
            first_valid_path = path
    return first_valid_path

def get_folders_matching_scheme(path_to_image_folders, pattern):
    target_folder_paths = []
    for root, dirs, files in os.walk(path_to_image_folders):

        match_result = pattern.match(os.path.basename(root))

        if match_result:  # basename matched the pattern
            target_folder_paths.append(root)

        # if os.path.basename(root) == target_folder_name_scheme:
        #     print 'Found Target Folder: ' + str(root)
        #     target_folder_paths.append(root)
    return target_folder_paths

def get_most_specific_parent_dir(possible_parent_dirs, possible_sub_dir):

    # print 'possible_sub_dir'
    # print possible_sub_dir

    current_parent_dir = ""

    for possible_parent_dir in possible_parent_dirs:

        # print 'possible_parent_dir'
        # print possible_parent_dir

        if is_subdir(possible_parent_dir, possible_sub_dir):

            # print 'is sub dir'

            if current_parent_dir == "":
                current_parent_dir = possible_parent_dir
            else:  # there is another parent dir already found

                result = is_subdir(current_parent_dir, possible_parent_dir)
                if result:
                    print("WARNING: FOUND A MORE SPECIFIC PARENT DIR")
                    print("current_parent_dir: " + current_parent_dir)
                    print("possible_parent_dir" + possible_parent_dir)
                    current_parent_dir = possible_parent_dir

    return current_parent_dir

def is_subdir(possible_parent_dir, possible_sub_dir):
    possible_parent_dir = os.path.realpath(possible_parent_dir)
    possible_sub_dir = os.path.realpath(possible_sub_dir)

    return possible_sub_dir.startswith(possible_parent_dir + os.sep)

def exist_files(file_list):
    return all(map(os.path.isfile, file_list))

def are_dirs_equal(idp_1, idp_2):
    fp_1_list = get_file_paths_in_dir(idp_1, base_name_only=True)
    fp_2_list = get_file_paths_in_dir(idp_2, base_name_only=True)

    return fp_1_list == fp_2_list


from itertools import combinations
import numpy

class ClusterNode(object):
  def __init__(self, vec, left, right, distance=0.0, count=1):
    self.left = left
    self.right = right
    self.vec = vec
    self.distance = distance
    self.count = count

  def extract_clusters(self, dist):
    '''Extract list of subtree clusters with distance < dist.'''
    if self.distance < dist:
      return [self]
    return self.left.extract_clusters(dist) + self.right.extract_clusters(dist)

  def get_cluster_elements(self):
    '''Return ids for elements in a cluster subtree.'''
    return self.left.get_cluster_elements() + self.right.get_cluster_elements()

  def get_height(self):
    '''Return the height of a node.'''
    return self.left.get_height() + self.right.get_height()

  def get_depth(self):
    '''Return the depth of a node.'''
    return max(self.left.get_depth(), self.right.get_depth()) + self.distance

  def draw(self, draw, x, y, s, imlist, im):
    '''Draw nodes recursively with iages for leaf nodes.'''

    # n^2 :-/
    h1 = int(self.left.get_height() * 20 / 2)
    h2 = int(self.right.get_height() * 20 / 2)
    top = y - (h1 + h2)
    bottom = y + (h1 + h2)

    draw.line((x, top + h1, x, bottom - h2), fill=(0, 0, 0))

    l1 = self.distance * s
    draw.line((x, top + h1, x + l1, top + h1), fill=(0, 0, 0))
    draw.line((x, bottom - h2, x + l1, bottom - h2), fill=(0, 0, 0))

    self.left.draw(draw, x + l1, top + h1, s, imlist, im)
    self.right.draw(draw, x + l1, bottom - h2, s, imlist, im)

class ClusterLeafNode(object):
  def __init__(self, vec, id):
    self.vec = vec
    self.id = id

  def extract_clusters(self, dist):
    return [self]

  def get_cluster_elements(self):
    return [self.id]

  def get_height(self):
    return 1

  def get_depth(self):
    return 0

  def draw(self, draw, x, y, s, imlist, im):
    from PIL import Image
    nodeim = Image.open(imlist[self.id])
    nodeim.thumbnail((20, 20))
    ns = nodeim.size
    im.paste(nodeim, [int(x), int(y - ns[1] // 2),
                      int(x + ns[0]), int(y + ns[1] - ns[1] // 2)])

def l2dist(v1, v2):
  return numpy.sqrt(numpy.sum((v1 - v2) ** 2))

def l1dist(v1, v2):
  return numpy.sum(numpy.abs(v1 - v2))

def hcluster(features, distfn=l2dist):
  '''Cluster the rows of features using hiearchical clustering.'''

  distances = {}

  node = [ClusterLeafNode(numpy.array(f), id=i) for i, f in enumerate(features)]

  # O(n^3)
  while len(node) > 1:
    closest = float('Inf')

    # Find closest pair.
    for ni, nj in combinations(node, 2):
      if (ni, nj) not in distances:
        distances[ni, nj] = distfn(ni.vec, nj.vec)

      d = distances[ni, nj]
      if d < closest:
        closest = d
        lowestpair = ni, nj
    ni, nj = lowestpair

    # Combine closest pair.
    new_vec = (ni.vec + nj.vec) / 2.0

    new_node = ClusterNode(new_vec, left=ni, right=nj, distance=closest)
    node.remove(ni)
    node.remove(nj)
    node.append(new_node)

  return node[0]

def draw_dendrogram(node, imlist, filename='out_clusters.png'):
  '''Draw a clsuter dendrogram and save it.'''
  from PIL import Image, ImageDraw

  rows = node.get_height() * 20
  cols = 1200

  s = float(cols - 150) / node.get_depth()

  im = Image.new('RGB', (cols, rows), (255, 255, 255))
  draw = ImageDraw.Draw(im)

  draw.line((0, rows/2, 20, rows/2), fill=(0, 0, 0))
  node.draw(draw, 20, rows / 2, s, imlist, im)
  im.save(filename)
  im.show()


#
# Fake questionnaire data generator.
#

# pylint: disable=superfluous-parens
import copy
import datetime
import logging
import random
import string
import time

from dateutil.parser import parse

from rdr_service import clock
from rdr_service.data_gen.generators.base_gen import BaseGen
from rdr_service.lib_fhir.fhirclient_1_0_6.models.coding import Coding
from rdr_service.lib_fhir.fhirclient_1_0_6.models.fhirdate import FHIRDate
from rdr_service.lib_fhir.fhirclient_1_0_6.models.fhirreference import FHIRReference
from rdr_service.lib_fhir.fhirclient_1_0_6.models.questionnaire import Questionnaire
from rdr_service.lib_fhir.fhirclient_1_0_6.models.questionnaireresponse import (
    QuestionnaireResponse,
    QuestionnaireResponseGroup,
    QuestionnaireResponseGroupQuestion,
    QuestionnaireResponseGroupQuestionAnswer,
)
from rdr_service.services.gcp_utils import gcp_make_auth_header
from rdr_service.services.system_utils import make_api_request

_logger = logging.getLogger("rdr_logger")

class QuestionnaireGen(BaseGen):
    """
  fake questionnaire response data generator.
  """

    _rdr_host = "localhost"

    _github_host = "raw.githubusercontent.com"
    _github_url = "all-of-us-terminology/api-payloads/master/questionnaire_payloads"
    _gen_url = "rdr/v1/SpecDataGen"
    _codebook = None  # stores the CodeBook object

    _module = None
    _module_codebook = None
    _module_questions = None

    _module_data = {
        "ConsentPII": {"file": "base_consent.json"},
        "TheBasics": {"file": "basics.json"},
        "EHRConsentPII": {"file": "ehr_consent.json"},
        "Lifestyle": {"file": "lifestyle.json"},
        "OverallHealth": {"file": "overall_health.json"},
    }
    # https://www.hl7.org/fhir/valueset-item-type.html
    _answer_types = [
        "boolean",
        "decimal",
        "integer",
        "date",
        "dateTime",
        "time",
        "string",
        "text",
        "url",
        "choice",
        "quantity",
    ]

    _participant_id = None
    _overrides = None
    _code_id = None

    def __init__(self, codebook, rdr_host=None):
        """
    Initialize the QuestionnaireGen object. Try to re-use to save on loading files.
    :param codebook: CodeBook object
    :param rdr_host: rdr service host
    """
        super(QuestionnaireGen, self).__init__()

        if rdr_host:
            self._rdr_host = rdr_host
        if not codebook:
            _logger.error("invalid codebook parameter.")

        self._codebook = codebook

    def new(self, module, participant_id, overrides=None):
        """
    Return a new QuestionnaireGen object with the assigned data.
    :param module: module name
    :param participant_id: participant id
    :param overrides: a dict of question id and answers
    :return: QuestionnaireGen object
    """
        if not module or not isinstance(module, str):
            _logger.error("invalid module name parameter")
            return None

        if not self._load_module(module):
            return None

        clone = self.__class__(self._codebook)
        clone._module = module
        clone._participant_id = participant_id
        clone._overrides = overrides
        # shallow copy some class objects so the the parent is updated
        clone._module_data = copy.copy(self._module_data)
        clone._module_questions = copy.copy(self._module_questions)
        clone._app_data = copy.copy(self._app_data)

        return clone

    def _load_module(self, module):
        """
    Load and prepare object for this module
    :param module: module name
    :return: True if module is loaded or False if not.
    """
        # Only reset if we need to.
        if self._module == module:
            return True

        self._module = module
        self._module_codebook = self._codebook.get_concept(module)
        self._module_questions = None
        retry = 5

        while retry > 0:
            self._module_questions = self._download_questions(module)
            if self._module_questions:
                break
            retry -= 1
            time.sleep(2)

        if not self._module_questions:
            _logger.error("failed to retrieve module questions.")
            return False
        # Load the questionnaire question codes into the module
        if "questions" not in self._module_data[module]:
            self._module_data[module]["questions"] = self._get_module_question_codes()

        return True

    def make_fhir_document(self):
        """
    Create a FHIR questionnaire response object, populate it with answers and return it.
    :return: fhir questionnaire response object
    """
        if not self._module_questions:
            raise NotImplementedError("only objects created with new() method may call this method.")

        q_resp = QuestionnaireResponse()
        q_resp.authored = FHIRDate(clock.CLOCK.now().isoformat())
        q_resp.status = "completed"
        q_resp.subject = FHIRReference({"reference": "Patient/{0}".format(self._participant_id)})
        q_resp.questionnaire = FHIRReference(
            {
                "reference": "Questionnaire/{0}/_history/{1}".format(
                    self._module_questions.id, self._module_questions.version
                )
            }
        )

        qnans = list()

        # additional answers for specific modules
        if self._module == "ConsentPII":
            qnans += self._answers_for_consentpii()
        if self._module == "TheBasics":
            qnans += self._answers_for_the_basics()
        if self._module == "EHRConsentPII":
            qnans += self._answers_for_ehr_consentpii()
        if self._module == "Lifestyle":
            qnans += self._answers_for_lifestyle()
        if self._module == "OverallHealth":
            qnans += self._answers_for_overall_health()

        # remove any invalid answers
        qnans = [x for x in qnans if x]

        # Handle question overrides
        if self._overrides and isinstance(self._overrides, list):
            qnans = self._apply_overrides(qnans, self._overrides)
        if len(qnans) == 0:
            return None

        q_group = QuestionnaireResponseGroup()
        q_group.question = qnans
        q_resp.group = q_group
        response = q_resp.as_json()
        # json_str = json.dumps(data)

        return response

    def _apply_overrides(self, qnans, overrides):
        """
    Apply any overrides to the final
    :param qnans: list of QuestionnaireResponseGroupQuestion objects
    :param overrides: list of tuples containing question code and answer
    :return: list of QuestionnaireResponseGroupQuestion objects
    """
        for (qn, an) in overrides:
            # check to make sure question code is in this module.
            if qn not in self._module_data[self._module]["questions"]:
                continue
            question = self._find_question(qn)
            qnan = self._answer_question(question, an)

            # look for existing question in list and remove it.
            index = -1
            for x in range(0, len(qnans)):
                if qnans[x].linkId == qnan.linkId:
                    index = x
                    break

            if index > -1:
                del qnans[index]
            qnans.append(qnan)

        return qnans

    def _make_contact_info(self, state_question=None):
        """
    Make first name, middle name, last name, phone number and email
    :param state_question: question object to use for looking up state ids
    :return: dict of values
    """
        data = dict()
        data["first"] = random.choice(self._app_data["first_names"])
        data["middle"] = random.choice(self._app_data["middle_names"])
        data["last"] = random.choice(self._app_data["last_names"])
        data["email"] = "{0}.{1}@fakeexample.com".format(data["first"], data["last"])
        data["phone"] = "{0:03}-{1:03}-{2:04}".format(
            random.randint(100, 999), random.randint(0, 999), random.randint(0, 9999)
        )
        street_info = random.choice(self._app_data["street_names"]).split("|")
        data["street1"] = street_info[0]
        if len(street_info) > 1:
            data["street2"] = street_info[1]
        data["city"] = random.choice(self._app_data["city_names"])
        data["zip"], data["state"] = random.choice(self._app_data["zipcodes"]).split(",")

        # attempt to look up the state choice value
        if state_question:
            states = self._get_question_choices(state_question)
            for state in states:
                if state.endswith(data["state"]):
                    data["state"] = state
                    break

        # Find a birth date between the 105 years and 12 years old.
        days_sub = random.randint((12 * 365), (105 * 365))
        data["dob"] = (clock.CLOCK.now() - datetime.timedelta(days=days_sub)).date()

        return data

    def _weighted_choice(self, choices):
        """
    Return a choice from a weighted set of choices.
    https://stackoverflow.com/questions/3679694/a-weighted-version-of-random-choice
    :param choices: a list of tuples containing choice and weight
    :return: weighted choice
    """
        total = sum(w for c, w in choices)
        r = random.uniform(0, total)
        upto = 0
        for c, w in choices:
            if upto + w >= r:
                return c
            upto += w
        assert False, "Shouldn't get here"

    def _answers_for_consentpii(self):
        """
    Create answers for ConsentPII module.
    :return: list of QuestionnaireResponseGroupQuestion objects
    """
        qnans = list()

        contact = self._make_contact_info(self._find_question("StreetAddress_PIIState"))
        qnans.append(self._answer_question(self._find_question("PIIName_First"), contact["first"]))
        qnans.append(self._answer_question(self._find_question("PIIName_Middle"), contact["middle"]))
        qnans.append(self._answer_question(self._find_question("PIIName_Last"), contact["last"]))
        qnans.append(self._answer_question(self._find_question("ConsentPII_EmailAddress"), contact["email"]))
        qnans.append(self._answer_question(self._find_question("PIIContactInformation_Email"), contact["email"]))
        qnans.append(self._answer_question(self._find_question("PIIContactInformation_Phone"), contact["phone"]))
        qnans.append(
            self._answer_question(self._find_question("ConsentPII_VerifiedPrimaryPhoneNumber"), contact["phone"])
        )
        qnans.append(self._answer_question(self._find_question("PIIAddress_StreetAddress"), contact["street1"]))
        qnans.append(self._answer_question(self._find_question("PIIAddress_StreetAddress2"), contact["street1"]))
        qnans.append(self._answer_question(self._find_question("StreetAddress_PIICity"), contact["city"]))
        qnans.append(self._answer_question(self._find_question("StreetAddress_PIIState"), contact["state"]))
        qnans.append(self._answer_question(self._find_question("StateOfResidence"), contact["state"]))
        qnans.append(self._answer_question(self._find_question("ReceiveCare_PIIState"), contact["state"]))
        qnans.append(self._answer_question(self._find_question("StreetAddress_PIIZIP"), contact["zip"]))
        qnans.append(
            self._answer_question(self._find_question("PIIBirthInformation_BirthDate"), contact["dob"].isoformat())
        )
        qnans.append(self._answer_question(self._find_question("ExtraConsent_WelcomeVideo"), clock.CLOCK.now()))
        qnans.append(self._answer_question(self._find_question("ExtraConsent_AgreeToConsent"), clock.CLOCK.now()))

        # q = self._find_question('Language')
        # q = self._find_question('Language_SpokenWrittenLanguage')

        optional = [
            "ExtraConsent_21YearsofAge",
            "ExtraConsent_19YearsofAge",
            "ExtraConsent_HealthRecordVideo",
            "ExtraConsent_CABoRSignature",
            "ExtraConsent__CABoRTodaysDate",
            "VeteransHealthAdministration",
            "ExtraConsent_KeepinginTouchVideo",
            "ExtraConsent_HealthDataVideo",
            "ExtraConsent_PhysicalMeausrementsVideo",
            "ExtraConsent_SamplesVideo",
            "ExtraConsent_DNAVideo",
            "ExtraConsent_FitnessTrackerVideo",
            "ExtraConsent_OtherHealthDataVideo",
            "ExtraConsent_DataSharingVideo",
            "ExtraConsent_RisktoPrivacyVideo",
            "ExtraConsent_EmailACopyToMe",
            "ExtraConsent_AllOfUsPurpose",
            "ExtraConsent_TakePartAllOfUs",
            "ExtraConsent_CanIWithdraw",
            "ExtraConsent_PrivacyGuaranteeQuestion",
            "ExtraConsent_ReadyOrNeedHelp",
            "ExtraConsent_AgreeToConsent",
            "ExtraConsent_Signature",
            "ExtraConsent_TodaysDate",
            "ConsentPII_HelpWithConsent",
            "ConsentPII_HelpWithConsentSignature",
        ]

        # Loop through the random questions and get answers for them.
        for qn_code in optional:
            qn = self._find_question(qn_code)
            if not qn:
                _logger.warning("question code not found in module [{0}].".format(qn_code))
                continue
            results = self._create_random_answer(qn)
            if results:
                for result in results:
                    qnans.append(self._answer_question(qn, result))

        return qnans

    def _answers_for_the_basics(self):
        """
    Create answers for TheBasics module.
    :return: list of QuestionnaireResponseGroupQuestion objects
    """
        qnans = list()

        # answer ethnicity question
        question = self._find_question("Race_WhatRaceEthnicity")
        answer = random.choice(self._get_question_choices(question))
        qnans.append(self._answer_question(question, answer))

        # see if we need to get an ethnicity sub category question answer
        if answer == "Asian_AsianSpecific":
            question = self._find_question("Asian_AsianSpecific")
            qnans.append(random.choice(question, random.choice(self._get_question_choices(question))))
        elif answer == "Black_BlackSpecific":
            question = self._find_question("Black_BlackSpecific")
            qnans.append(random.choice(question, random.choice(self._get_question_choices(question))))
        elif answer == "Hispanic_HispanicSpecific":
            question = self._find_question("Hispanic_HispanicSpecific")
            qnans.append(random.choice(question, random.choice(self._get_question_choices(question))))
        elif answer == "MENA_MENASpecific":
            question = self._find_question("MENA_MENASpecific")
            qnans.append(random.choice(question, random.choice(self._get_question_choices(question))))
        elif answer == "NHPI_NHPISpecific":
            question = self._find_question("NHPI_NHPISpecific")
            qnans.append(random.choice(question, random.choice(self._get_question_choices(question))))
        elif answer == "White_WhiteSpecific":
            question = self._find_question("White_WhiteSpecific")
            qnans.append(random.choice(question, random.choice(self._get_question_choices(question))))
        if random.random() > 0.6:
            contact = self._make_contact_info(self._find_question("PersonOneAddress_PersonOneAddressState"))

            qnans.append(
                self._answer_question(self._find_question("SecondaryContactInfo_PersonOneFirstName"), contact["first"])
            )
            qnans.append(
                self._answer_question(
                    self._find_question("SecondaryContactInfo_SecondContactsMiddleInitial"), contact["middle"][:1]
                )
            )
            qnans.append(
                self._answer_question(self._find_question("SecondaryContactInfo_PersonOneLastName"), contact["last"])
            )
            qnans.append(
                self._answer_question(self._find_question("SecondaryContactInfo_PersonOneTelephone"), contact["phone"])
            )
            qnans.append(
                self._answer_question(self._find_question("SecondaryContactInfo_PersonOneEmail"), contact["email"])
            )

            qnans.append(
                self._answer_question(
                    self._find_question("SecondaryContactInfo_PersonOneAddressOne"), contact["street1"]
                )
            )
            qnans.append(
                self._answer_question(
                    self._find_question("SecondaryContactInfo_PersonOneAddressTwo"), contact["street1"]
                )
            )
            qnans.append(
                self._answer_question(self._find_question("PersonOneAddress_PersonOneAddressCity"), contact["city"])
            )
            qnans.append(
                self._answer_question(self._find_question("PersonOneAddress_PersonOneAddressState"), contact["state"])
            )
            qnans.append(
                self._answer_question(self._find_question("PersonOneAddress_PersonOneAddressZipCode"), contact["zip"])
            )
        else:
            if random.random() > 0.5:
                qnans.append(
                    self._answer_question(self._find_question("SecondaryContactInfo_PersonOnePreferNotToAnswer"), True)
                )

        # figure out work place location
        if random.random() > 0.04:
            # "EmploymentWorkAddress_AddressLineOne", "EmploymentWorkAddress_AddressLineTwo",
            # "EmploymentWorkAddress_City", "EmploymentWorkAddress_State",
            # "EmploymentWorkAddress_ZipCode", "EmploymentWorkAddress_Country",
            # "Employment_EmploymentWorkAddress",
            pass

        # Make a weighted choice for gender
        question = self._find_question("Gender_GenderIdentity")
        gender_codes = self._get_question_choices(question)
        gender_choices = list()

        for code in gender_codes:
            gender_choices.append(
                (code, 5.0) if code in ["GenderIdentity_Man", "GenderIdentity_Woman"] else (code, 0.08)
            )

        gender = self._weighted_choice(gender_choices)
        suffix = gender.split("_")[1]
        qnans.append(self._answer_question(question, gender))

        # Make a weighted choice for birth sex
        question = self._find_question("BiologicalSexAtBirth_SexAtBirth")
        sex_codes = self._get_question_choices(question)
        sex_choices = list()

        for code in sex_codes:
            sex_choices.append((code, 9.0) if suffix in code else (code, 0.4))
        qnans.append(self._answer_question(question, self._weighted_choice(sex_choices)))
        # Make a weighted choice about sexual orientation.
        question = self._find_question("TheBasics_SexualOrientation")
        orient_codes = self._get_question_choices(question)
        orient_choices = list()

        for code in orient_codes:
            orient_choices.append((code, 9.0) if code == "SexualOrientation_Straight" else (code, 0.5))
        qnans.append(self._answer_question(question, self._weighted_choice(orient_choices)))
        # Make a weighted choice for education.
        question = self._find_question("EducationLevel_HighestGrade")
        edu_codes = self._get_question_choices(question)
        edu_choices = list()

        for code in edu_codes:
            edu_choices.append(
                (code, 3.5)
                if code
                in ["HighestGrade_TwelveOrGED", "HighestGrade_CollegeOnetoThree", "HighestGrade_CollegeGraduate"]
                else (code, 0.5)
            )
        qnans.append(self._answer_question(question, self._weighted_choice(edu_choices)))
        # Make annual income choice.
        question = self._find_question("Income_AnnualIncome")
        income_choices = self._get_question_choices(question)
        qnans.append(self._answer_question(question, random.choice(income_choices)))

        optional = [
            "TheBasics_CountryBornTextBox",
            "ActiveDuty_AvtiveDutyServeStatus",
            "MaritalStatus_CurrentMaritalStatus",
            "LivingSituation_HowManyPeople",
            "LivingSituation_PeopleUnder18",
            "Insurance_HealthInsurance",
            "HealthInsurance_InsuranceTypeUpdate",
            "OtherHealthPlan_FreeText",
            "Employment_EmploymentStatus",
            "HomeOwn_CurrentHomeOwn",
            "LivingSituation_CurrentLiving",
            "LivingSituation_LivingSituationFreeText",
            "LivingSituation_HowManyLivingYears",
            "LivingSituation_StableHouseConcern",
        ]

        # Loop through the random questions and get answers for them.
        for qn_code in optional:
            qn = self._find_question(qn_code)
            if not qn:
                _logger.warning("question code not found in module [{0}].".format(qn_code))
                continue
            results = self._create_random_answer(qn)
            if results:
                for result in results:
                    qnans.append(self._answer_question(qn, result))

        return qnans

    def _answers_for_ehr_consentpii(self):
        """
    Create additional answers for EHRConsentPII module
    :return: list of QuestionnaireResponseGroupQuestion objects
    """
        qnans = self._answer_all_module_questions()

        question = self._find_question("EHRConsentPII_JoinAllOfUs")
        qnans.append(self._answer_question(question, True))

        return qnans

    def _answers_for_lifestyle(self):
        """
    Create additional answers for Lifestyle module
    :return: list of QuestionnaireResponseGroupQuestion objects
    """
        qnans = self._answer_all_module_questions()
        return qnans

    def _answers_for_overall_health(self):
        """
    Create additional answers for OverallHealth module
    :return: list of QuestionnaireResponseGroupQuestion objects
    """
        qnans = self._answer_all_module_questions()
        return qnans

    def _create_random_answer(self, question):
        """
    Create a random answer for the given question
    :param question: question object from self.module_questions object
    :return: list of answer values
    """
        if not question:
            return None

        answers = list()
        qn_code = question.concept[0].code

        if not qn_code:
            _logger.warning("link id is not a question [{0}].".format(question.linkId))
            return answers

        # see if this question is listed in the answer_specs data
        for item in self._app_data["answer_specs"]:
            if qn_code in item["question_code"]:
                if random.random() > (float(item["num_participants"]) / 100.0):

                    # if this question is answered more than once, choose a random number of answers.
                    na = float(item["num_answers"])
                    nqr = float(item["num_questionnaire_responses"])
                    count = 1

                    if na > nqr:
                        rand_val = random.random() * (na / nqr)
                        if rand_val > 1:
                            # Set the answer count to 2 or more
                            count = int(1 + max(1, rand_val))
                    # pylint: disable=unused-variable
                    for x in range(0, count):
                        answer = self._choose_random_answer(question, item)
                        if answer:
                            answers.append(answer)

                return answers

        # decide if we are going to answer this question or not
        if random.random() < 0.7:
            return answers

        # choose a random answer for all other questions
        answers.append(self._choose_random_answer(question))

        return answers

    def _choose_random_answer(self, question, item=None):
        """
    Generate an answer to the spec question.
    :param question: question object from self.module_questions object
    :param item: item from self._data_lists['answer_specs']
    :return: answer
    """
        if question.type not in self._answer_types:
            _logger.debug("unhandled or invalid question answer type [{0}].".format(question.type))
            return None

        answer = None

        if question.type == "boolean":
            if item:  # handle spec item
                if int(item["boolean_answer_count"]) > 0:
                    answer = random.random() < 0.5
            else:
                answer = random.random() < 0.5

        elif question.type == "decimal":
            if item:
                if int(item["decimal_answer_count"]) > 0:
                    answer = round(
                        random.uniform(float(item["min_decimal_answer"]), float(item["max_decimal_answer"])), 1
                    )
            else:
                answer = round(random.uniform(0.0, 20.0), 1)

        elif question.type == "integer":
            if item:
                if int(item["integer_answer_count"]) > 0:
                    answer = random.randint(int(item["min_integer_answer"]), int(item["max_integer_answer"]))
            else:
                answer = random.randint(0, 100)

        elif question.type == "date":
            if item:
                if int(item["date_answer_count"]) > 0:
                    min_date = parse(item["min_date_answer"])
                    max_date = parse(item["max_date_answer"])
                    days_diff = (max_date - min_date).days
                    answer = (min_date + datetime.timedelta(days=random.randint(0, days_diff))).isoformat()
            else:
                days_sub = random.randint(1, (5 * 365))
                answer = (clock.CLOCK.now() - datetime.timedelta(days=days_sub)).isoformat()

        elif question.type == "dateTime":
            if item:
                if int(item["datetime_answer_count"]) > 0:
                    min_date = parse(item["min_datetime_answer"])
                    max_date = parse(item["max_datetime_answer"])
                    seconds_diff = (max_date - min_date).total_seconds()
                    answer = (min_date + datetime.timedelta(seconds=random.randint(0, seconds_diff))).isoformat()
            else:
                days_sub = random.randint(1, (5 * 365))
                answer = (clock.CLOCK.now() - datetime.timedelta(days=days_sub)).isoformat()

        elif question.type == "time":
            pass

        elif question.type == "string":
            if item:
                if int(item["string_answer_count"]) > 0:
                    answer = " ".join(
                        [random.choice(self._app_data["latin_words"]) for _ in range(random.randint(1, 5))]
                    )
            else:
                answer = " ".join([random.choice(self._app_data["latin_words"]) for _ in range(random.randint(1, 5))])

        elif question.type == "text":
            answer = " ".join([random.choice(self._app_data["latin_words"]) for _ in range(random.randint(1, 5))])

        elif question.type == "choice":
            choices = self._get_question_choices(question)
            if item:
                if int(item["code_answer_count"]) > 0:
                    answer = random.choice(choices)
            else:
                answer = random.choice(choices)

        elif question.type == "url" or question.type == "uri":
            bucket = "gs://notarealbucket.example.com/{0}".format(
                "".join([random.choice(string.lowercase) for _ in range(20)])
            )
            if item:
                if int(item["uri_answer_count"]) > 0:
                    answer = bucket
            else:
                answer = bucket

        elif question.type == "quantity":
            pass

        return answer

    def _answer_question(self, question, answer, quantity=None):
        """
    Setup a question answer.
    :param question: question object from self.module_questions object
    :param answer: answer value
    :param quantity: models.quantity.Quantity object
    :return: QuestionnaireResponseGroupQuestion object
    """
        if question is None:
            _logger.debug("code not found, skip this question")
            return None
        if question.type not in self._answer_types:
            _logger.debug("unhandled or invalid question answer type [{0}].".format(question.type))
            return None

        # setup answer object
        an = QuestionnaireResponseGroupQuestionAnswer()

        # store answer
        if question.type == "boolean":
            an.valueBoolean = False
            if isinstance(answer, str):
                if answer.lower() in ["yes", "true"]:
                    an.valueBoolean = True
            if isinstance(answer, bool):
                an.valueBoolean = answer

        elif question.type == "decimal":
            an.valueDecimal = float(answer)

        elif question.type == "integer":
            an.valueInteger = int(answer)

        elif question.type == "date":
            if isinstance(answer, str):
                an.valueDate = FHIRDate(answer)
            else:
                an.valueDate = FHIRDate(answer.isoformat())

        elif question.type == "dateTime":
            if isinstance(answer, str):
                an.valueDateTime = FHIRDate(answer)
            else:
                an.valueDateTime = FHIRDate(answer.isoformat())

        elif question.type == "time":
            if isinstance(answer, str):
                an.valueTime = FHIRDate(answer)
            else:
                an.valueTime = FHIRDate(answer.isoformat())

        elif question.type == "string":
            an.valueString = answer

        elif question.type == "text":
            an.valueString = answer

        elif question.type == "url" or question.type == "uri":
            an.valueUri = answer

        elif question.type == "choice":
            if answer in self._get_question_choices(question):
                an.valueCoding = Coding(
                    jsondict={
                        "code": answer,
                        "system": "http://terminology.pmi-ops.org/CodeSystem/ppi",
                        "display": answer,
                    }
                )

        elif question.type == "quantity":
            an.valueQuantity = quantity

        if not an:
            return None

        qn = QuestionnaireResponseGroupQuestion()
        qn.linkId = question.linkId
        qn.answer = list()
        qn.answer.append(an)

        return qn

    def _download_questions_from_github(self, module):
        """
    Download fhir questionnaires from github.
    :return: fhir questionnaire object
    """
        _host = "raw.githubusercontent.com"
        _url = "all-of-us-terminology/api-payloads/master/questionnaire_payloads/"

        for key, data in list(self._module_data.items()):
            if key.lower() == module.lower():

                code, resp = make_api_request(_host, "{0}/{1}".format(_url, data["file"]))

                if code != 200:
                    _logger.error("failed to get module questions from github.")
                    return None

                questions = Questionnaire(resp, strict=False)
                if questions:
                    return questions

        _logger.error("failed to parse the module questions data.")
        return None

    def _download_questions(self, module):
        """
    Download fhir questionnaire from rdr service.
    :param module: questionnaire module name
    :return: fhir questionnaire object
    """
        data = dict()
        data["api"] = "Questionnaire?concept={0}".format(module)
        data["timestamp"] = clock.CLOCK.now().isoformat()
        data["method"] = "GET"

        code, resp = make_api_request(
            self._rdr_host, self._gen_url, req_type="POST", json_data=data, headers=gcp_make_auth_header()
        )

        if code != 200:
            _logger.error("failed to get module questionnaire [Http {0}: {1}.".format(code, resp))
            return None

        questions = Questionnaire(resp, strict=False)
        if questions:
            return questions

        _logger.error("failed to parse the module questionnaire data.")
        return None

    def _get_module_question_codes(self):
        """
    Return all of the question code ids in the module
    :return: list
    """
        codes = list()

        for item in self._module_questions.group.question:
            qn_code = item.concept[0].code

            if qn_code == "null":
                _logger.warning("question code id set to null in module.")
                continue

            codes.append(qn_code)

        return codes

    def _get_question_choices(self, question):
        """
    Return a list of possible choices for the given question
    :param question: question object from self.module_questions object
    :return: list
    """
        if question.type != "choice":
            _logger.warning("question type is not a choice question type.")
            return list()

        choices = list()
        for choice in question.option:
            choices.append(choice.code)

        return choices

    def _find_question(self, question_code):
        """
    Get the specific question from the available module questions.
    :param question_code: question code id string
    :return: QuestionnaireGroupQuestion object
    """
        if not self._module_questions:
            _logger.error("module questions not initialized.")
            return None

        for question in self._module_questions.group.question:
            if not question.concept[0].code:
                continue
            if "Language" in question.concept[0].code:
                print((question.concept[0].code))
            if question_code == question.concept[0].code:
                return question
        _logger.warning("question not found for question code {}.".format(question_code))
        return None

    def _answer_all_module_questions(self):
        """
    Randomly answer all module questions.
    :return: list of QuestionnaireResponseGroupQuestion objects
    """
        qnans = list()

        for question in self._module_questions.group.question:
            qn_code = question.concept[0].code
            # print(qn_code)

            if qn_code == "null":
                _logger.warning("question code id set to null in module.")
                continue

            # create a random answer for the spec question.
            results = self._create_random_answer(question)
            for result in results:
                qnans.append(self._answer_question(question, result))

        return qnans


import heapq
T = int(input())
res = []
for case in range(T):
    N = int(input())
    nums = [int(x) for x in input().split()]
    h = 0
    heap = []

    for i, n in enumerate(nums):
        if n > h:
            heapq.heappush(heap, n)
            if len(heap) >= h + 1:
                while heap and heap[0] <= h + 1:
                    heapq.heappop(heap)
                h += 1
        nums[i] = str(h)
    print('Case #{}: {}'.format(case + 1, ' '.join(nums)))


# Definition for singly-linked list.
# class ListNode:
#     def __init__(self, x):
#         self.val = x
#         self.next = None

class Solution:
    def hasCycle(self, head: ListNode) -> bool:
        turtle = rabbit = head

        while rabbit and rabbit.next:
            turtle = turtle.next
            rabbit = rabbit.next.next
            if turtle == rabbit:
                return True

        return False


import numpy as np
import os.path as path
from scipy import spatial

class Glove:
    """
    Class to handle the glove Embeddings, load the pretrained
    embeddings from the directory and return embedding
    vector for words.
    """
    def __init__(self):
        self.embeddings_dict = {}
        self.n_dimension = 0
        self.average = None

    def load(self):
        """
        Load the embedding vector from directory
        and store the whole embedding as a dictionary
        :return: none
        """
        with open(path.dirname(path.abspath(__file__)) + "pretrained/glove.6B.300d.txt", 'r', encoding="utf-8") as f:
            for line in f:
                values = line.split()
                word = values[0]
                vector = np.asarray(values[1:], "float32")
                self.embeddings_dict[word] = vector

        first_key = next(iter(self.embeddings_dict))
        self.n_dimension = len(self.embeddings_dict[first_key])
        self.calculate_average_vec()

    def calculate_average_vec(self):
        n_vec = len(self.embeddings_dict)
        vecs = np.zeros((n_vec, self.n_dimension), dtype=np.float32)
        for i, key in enumerate(self.embeddings_dict):
            vecs[i] = self.embeddings_dict[key]

        self.average = np.mean(vecs, axis=0)

    def find_closest_words(self, term, limit=5):
        embedding = self.embeddings_dict[term]
        all_words = sorted(self.embeddings_dict.keys(),
                      key=lambda word: spatial.distance.euclidean(self.embeddings_dict[word], embedding))
        return all_words[:limit]

    def get_embedding_val(self, term, average_on_none=False):
        """
        Get the vector of each term
        :param term: a string, for which it will search the dictonary for embedding vector
        :param average_on_none: Boolean, if true will return average vector if not found on Dictionary else
        return None
        :return: Vector or None
        """
        try:
            val = self.embeddings_dict[term.lower()]
        except KeyError:
            if average_on_none:
                val = self.average
            else:
                val = None
        return val


from django.core.urlresolvers import reverse
from django.http import Http404
from django.shortcuts import render
from django.shortcuts import redirect
from django.shortcuts import HttpResponseRedirect
from django.conf import settings
from thought.thought import *
from thought.dicontainer import CONTAINER

def index(request):
    tags = CONTAINER.getTagManager().getTags()
    tagMax = tags and max([tag.count for tag in tags]) or 0
    thoughts = CONTAINER.getThoughtManager().latest(settings.THOUGHTS_PER_PAGE)
    showMore = len(thoughts) == settings.THOUGHTS_PER_PAGE
    loadMoreUrl = reverse("latestPage", args=[1])

    return render(request, "thought/index.html",
            {'thoughts': thoughts, 'tags': tags, 'tagMax': tagMax,
             'showMore': showMore, 'loadMoreUrl': loadMoreUrl})

def latestPage(request, page):
    page = int(page)
    thoughts = CONTAINER.getThoughtManager().latest(
        settings.THOUGHTS_PER_PAGE,
        skip=page * settings.THOUGHTS_PER_PAGE)
    showMore = len(thoughts) == settings.THOUGHTS_PER_PAGE
    loadMoreUrl = reverse("latestPage", args=[page + 1])

    return render(request, "thought/thoughts_page.html", {
        'thoughts': thoughts, "showMore": showMore,
        'loadMoreUrl': loadMoreUrl
    })

def post(request):
    error = None
    thought = Thought()

    if request.method == 'POST':
        thought.text = request.POST.get('text')

        if thought.text:
            thought.date = datetime.now()
            thought.tags = getTagsList(request.POST.get('tags'))
            CONTAINER.getThoughtManager().create(thought)

            return redirect('index')
        else:
            error = "Thought shouldn't be empty, otherwise who may need it?"

    return render(request, "thought/post.html", {'thought': thought, 'error': error})

def edit(request, id):
    error = None
    thought = CONTAINER.getThoughtManager().get(id)

    if thought is None:
        raise Http404()

    if request.method == 'POST':
        thought.text = request.POST.get('text')
        if thought.text:
            thought.tags = getTagsList(request.POST.get('tags'))
            CONTAINER.getThoughtManager().save(thought)

            return redirect('index')
        else:
            error = "Thought shouldn't be empty, otherwise who may need it?"

    tags = (thought.get_tags() and ", ".join(thought.get_tags()) or "")
    return render(request, "thought/edit.html", {'thought': thought, 'tags': tags, 'error': error})

def thought(request, id):
    thought = CONTAINER.getThoughtManager().get(id)
    return render(request, "thought/thought.html", {'thought': thought})

def remove(request, id):
    CONTAINER.getThoughtManager().deleteById(id)
    return HttpResponseRedirect(reverse('index'))

def tag(request, tag):
    tags = CONTAINER.getTagManager().getTags()
    tagMax = tags and max([each.count for each in tags]) or 0
    thoughts = CONTAINER.getThoughtManager().searchByTag(tag, settings.THOUGHTS_PER_PAGE)
    showMore = len(thoughts) == settings.THOUGHTS_PER_PAGE
    loadMoreUrl = reverse("tagPage", args=[tag, 1])

    return render(request, "thought/tag.html", {
        "tag": tag, 'thoughts': thoughts, 'tags': tags,
        'tagMax': tagMax, 'showMore': showMore,
        'loadMoreUrl': loadMoreUrl
    })

def tagPage(request, tag, page):
    page = int(page)
    thoughts = CONTAINER.getThoughtManager().searchByTag(
        tag, settings.THOUGHTS_PER_PAGE,
        skip=page * settings.THOUGHTS_PER_PAGE)

    showMore = len(thoughts) == settings.THOUGHTS_PER_PAGE
    loadMoreUrl = reverse("tagPage", args=[tag, page + 1])

    return render(request, "thought/thoughts_page.html", {
        'thoughts': thoughts,
        'showMore': showMore,
        'loadMoreUrl': loadMoreUrl
    })

def appSettings(request):
    return render(request, "thought/settings.html")

def getTagsList(tagsText):
    tags = parseTags(tagsText)
    removeEmptyTag(tags)
    tags.sort()

    return tags

def parseTags(tagsText):
    tags = [tag.strip().lower() for tag in tagsText.split(",")]
    return prepareTags(tags)

def prepareTags(tags):
    return [tag.lower().replace("/", "") for tag in tags]

def removeEmptyTag(tags):
    try:
        tags.remove("")
    except ValueError:
        pass


import asyncio
import discord
import os
import utils
from discord.ext import commands

path = os.path.abspath(os.path.join(os.getcwd(), os.pardir))
token = open("token.txt").read()

intents = discord.Intents.all()
intents.members = True
intents.presences = True
intents.messages = True

#bot = commands.Bot(command_prefix=',', intents=intents)

class MyBot(commands.Bot):
    async def setup_hook(self):
        bot.remove_command('help')
        await load_extensions()
        print("extensions loaded")

async def load_extensions():
    for file in os.listdir("cogs"):
        if file.endswith(".py"):
            name = file[:-3]
            await bot.load_extension('cogs.' + name)

bot = MyBot(command_prefix=',', intents=intents)
bot.run(token)

# @bot.event
# async def on_ready():
#     print("bird up")
#     for guild in bot.guilds:
#         for member in guild.members:
#             if utils.check_info(member) == "Error":
#                 utils.add_user_entry(str(member), member.id, member.guild.id)


import os
import sys
import time
import datetime
import numpy as np

import tensorflow as tf
from tensorflow.keras.preprocessing.image import load_img, img_to_array, ImageDataGenerator
from tensorflow.keras.layers import Input
import tensorflow.keras.backend as K

from absl import app, logging, flags
from absl.flags import FLAGS

from dataset_manager import Market1501
from models.osnet import OSNet
from losses import CrossEntropyLabelSmooth, DeepSupervision
from models.models import get_optimizer
# Importing dependencies

# adding cmd args
# optimization Options
flags.DEFINE_string('optim', 'sgd', 'optimization algorithm')
flags.DEFINE_integer('max_epoch', 350, 'maximum epoch to run')
flags.DEFINE_integer('start_epoch', 0, 'manual epoch number')
flags.DEFINE_integer('train_batch', 64, 'train batch size')
flags.DEFINE_integer('test_batch', 32, 'test batch size')
flags.DEFINE_float('lr', 0.065, 'initial learning rate')
flags.DEFINE_integer('stepsize', 20, 'stepsize to decay learning rate')
flags.DEFINE_float('gamma', 0.1, 'learning rate decay')
#flags.DEFINE_enum('lr_sched', 'linear', ['linear', 'exp'], 'linear : linear Way' 'exp : Exponetial Way')
flags.DEFINE_float('weight_decay', 5e-04, 'weight decay')

# Miscs
flags.DEFINE_integer('print_freq', 10, 'print frequency') #10
flags.DEFINE_integer('seed', 1, 'manual seed')
flags.DEFINE_string('resume', '', 'PATH')
flags.DEFINE_bool('evaluate', False, 'evaluation only')
flags.DEFINE_integer('eval_step', 1, 'run evaluation for every N epochs')
flags.DEFINE_integer('start_eval', 0, 'start to evaluate after specific epoch')
flags.DEFINE_string('save_dir', 'logs', 'save directory')
flags.DEFINE_bool('use_cpu', False, 'evaluation only')
flags.DEFINE_string('gpu_devices', '0', 'gpu device ids for CUDA_VISIBLE_DEVICES')

train_datagen = ImageDataGenerator(
            horizontal_flip=True,
            featurewise_std_normalization=True,
            # translation is remaining
            )
test_datagen = ImageDataGenerator(
            featurewise_std_normalization=True,
            # translation is remaining
        )
def transform(data, test=False):
    imgs_path, rest = data[:,0], data[:,1:].astype(np.int32)
    imgs = np.array(list(map(lambda x: img_to_array(load_img(x)), imgs_path)))
    if not test:
        train_datagen.fit(imgs)
        return train_datagen, imgs, rest
    else:
        test_datagen.fit(imgs)
        return test_datagen, imgs, rest

def evaluate(distmat, q_pids, g_pids, q_camids, g_camids, max_rank=50):
    num_q, num_g = distmat.shape
    if num_g < max_rank:
        max_rank = num_g
        print("Note: number of gallery samples is quite small, got {}".format(num_g))
    indices = np.argsort(distmat, axis = 1)
    matches = (g_pids[indices] == q_pids[:, np.newaxis]).astype(np.int32)

    # compute cmc curve for each query
    all_cmc = []
    all_AP = []
    num_valid_q = 0. # number of valid query

    for q_idx in range(num_q):
        # get query pid and camid
        q_pid = q_pids[q_idx]
        q_camid = q_camids[q_idx]

        # remove gallery sample that have the same pid and camid with query
        order = indices[q_idx]
        remove = (g_pids[order] == q_pid) & (g_camids[order] == q_camid)
        keep = np.invert(remove)

        # compute cmc curve
        orig_cmc = matches[q_idx][keep]
        if not np.any(orig_cmc):
            # this condition is true when query identify does not appear in gallery
            continue

        cmc = orig_cmc.cumsum()
        cmc[cmc > 1] = 1

        all_cmc.append(cmc[:max_rank])
        num_valid_q += 1.

        # compute average precision
        # reference: https://en.wikipedia.org/wiki/Evaluation_measures_(information_retrieval)#Average_precision
        num_rel = orig_cmc.sum()
        tmp_cmc = orig_cmc.cumsum()
        tmp_cmc = [x / (i+1.) for i, x in enumerate(tmp_cmc)]
        tmp_cmc = np.asarray(tmp_cmc) * orig_cmc
        AP = tmp_cmc.sum() / num_rel
        all_AP.append(AP)

    assert num_valid_q > 0, "Error: all query identities do not appear in gallery"

    all_cmc = np.asarray(all_cmc).astype(np.float32)
    all_cmc = all_cmc.sum(0) / num_valid_q
    mAP = np.mean(all_AP)

    return all_cmc, mAP

def main(_argv):
    # setting up cuda visibility
    tf.random.set_seed(FLAGS.seed)
    if not FLAGS.use_cpu:
        os.environ['TF_CUDNN_DETERMINISTIC'] = '1'
        os.environ['CUDA_VISIBLE_DEVICES'] = FLAGS.gpu_devices
        physical_devices = tf.config.experimental.list_physical_devices('GPU')
        for physical_device in physical_devices:
            tf.config.experimental.set_memory_growth(physical_device, True)
        use_gpu = True
    else:
        use_gpu = False

    now = datetime.datetime.now().strftime("%m_%d_%Y__%H_%M_%S")
    # gather and manage dataset from dataset manager
    print("Initializing dataset")
    dataset = Market1501()
    print("Dataset initialized")
    #print(dataset.train[0])
    # create data loader for train, query & gallery
    print("processing dataset")
    train_datagen, train_imgs, train_labels = transform(np.array(dataset.train))
    print("train done")
    query_datagen, query_imgs, query_labels = transform(np.array(dataset.query), test=True)
    print("query done")
    gallery_datagen, gallery_imgs, gallery_labels = transform(np.array(dataset.gallery), test=True)
    print("gallery done")

    # creating model, criterion, optimizer, stepLR if it's present
    model = OSNet(classes=dataset.num_train_pids, loss_type="triplet")
    raw_output = model(Input(shape=(256, 128, 3)), training=True)

    #total_para = np.sum([K.count_params(w) for w in model.trainable_weights]) + np.sum([K.count_params(w) for w in model.non_trainable_weights])
    print(model.summary())
    """
    imgs, labels = train_datagen.flow(train_imgs, train_labels, batch_size=FLAGS.train_batch)[0]
    print(imgs.shape, labels.shape)
    """

    criterion = CrossEntropyLabelSmooth(num_classes= dataset.num_train_pids)
    optimizer = get_optimizer(FLAGS.optim, FLAGS.lr, FLAGS.weight_decay)
    lr_decay_epoches = [150, 225, 300]
    start_epoch = FLAGS.start_epoch
    # conditional branch for resuming_training, evaluate
    if FLAGS.resume != '':
        print("Loading checkpoints from {}".format(FLAGS.resume))
        model.load_weights(FLAGS.resume)
        name = FLAGS.resume.split("/")[-1]
        start_epoch = int(name.split("_")[2].replace("epoch", ""))
        now += "_resumed"
        #print(start_epoch)
    summary_writer = tf.summary.create_file_writer('logs/{}'.format(now))
    # initialize start_time, trian_time, best_rank1, best_epoch
    start_time = time.time()
    train_time = 0
    best_rank1 = -np.inf
    best_epoch = 0
    print("==> Start training")

    # iterate epoch
    for epoch in range(start_epoch, FLAGS.max_epoch):
        print("\tCurrent epoch:{}".format(epoch+1))
        tf.summary.trace_on(graph=True)
        # initialize train_time
        start_train_time = time.time()
        # make 1 epoch with training set / train
        losses, accuracy = train(epoch, model, criterion, optimizer, train_datagen, train_imgs, train_labels)
        train_time += round(time.time() - start_train_time)
        # calculate train_time

        # make scheduler steps as per epoch condition
        if FLAGS.stepsize > 0 and epoch+1 in lr_decay_epoches:
            optimizer.lr = optimizer.lr - (optimizer.lr * 0.1)

        # make evaluation/validation by checking current epoch+1 is multiple of 10
        if (epoch+1) % 10 == 0:                      # 10
            print("==> Test")
            rank1, mAP = test(model, query_datagen, query_imgs, query_labels, gallery_datagen, gallery_imgs, gallery_labels)

            is_best = rank1 > best_rank1
            if is_best:
                best_rank1 = rank1
                best_epoch = epoch + 1

            with summary_writer.as_default():
                tf.summary.scalar('rank-1', rank1, step=epoch+1)
                tf.summary.scalar('mAP', mAP, step=epoch+1)
            model.save_weights(
                'checkpoints/{}/osnet_train_epoch{}_rank1{}.tf'.format(now, epoch, rank1))

        with summary_writer.as_default():
            tf.summary.scalar('loss', losses.result(), step=epoch+1)
            tf.summary.scalar('accuracy', accuracy.result(), step=epoch+1)
            tf.summary.scalar('lr:{}'.format(FLAGS.optim), optimizer.lr, step=epoch+1)
            tf.summary.trace_export('name', step=0)

    print("==> Best Rank-1 {:.1%}, achieved at epoch {}".format(best_rank1, best_epoch))

    elapsed = round(time.time() - start_time)
    elapsed = str(datetime.timedelta(seconds=elapsed))
    train_time = str(datetime.timedelta(seconds=train_time))
    print("Finished. Total elapsed time (h:m:s): {}. Training time (h:m:s): {}.".format(elapsed, train_time))
            # make batch prediction with gallery set and query / make test
            # check rank1 if it is best then change best_rank1 and epoch otherwise leave

            # saves model parameter
    # print out result with best rank and epoch
    # print total time for training

def train(epoch, model, criterion, optimizer, train_datagen, train_imgs, train_labels):
    """
    # creating losses_paras, batch_time_paras, data_time_paras with AverageMeter
    # make train the model
    # total end time

    # iterate as per batches
        # converts images and pids from cuda

        # update data_time with current time

        # make batch prediction with train_images
        # calculate loss
        # optimizer zero grad
        # backpropogate loss
        # optimizer step

        # update batch_time, end, losses
        # print epoch status with epoch val, learning rate, time, data, loss
    Args:
        epoch ([type]): [description]
        model ([type]): [description]
        criterion ([type]): [description]
        optimizer ([type]): [description]
        train_datagen ([type]): [description]
        train_imgs ([type]): [description]
        train_labels ([type]): [description]
    """
    accuracy = tf.keras.metrics.CategoricalAccuracy('accuracy', dtype=tf.float32)
    losses = tf.keras.metrics.Mean('loss', dtype=tf.float32)
    batch_time = tf.keras.metrics.Mean('train_batch_time', dtype=tf.float32)
    data_time = tf.keras.metrics.Mean('data_time', dtype=tf.float32)

    end = time.time()
    trian_max_batches = train_imgs.shape[0] // FLAGS.test_batch if train_imgs.shape[0] % FLAGS.train_batch == 0 else (train_imgs.shape[0] // FLAGS.train_batch) + 1
    for batch_idx, (imgs, labels) in enumerate(train_datagen.flow(train_imgs, train_labels, batch_size=FLAGS.train_batch)):
        #print("batch id:{}".format(batch_idx))
        pids, camids = labels[:,0], labels[:,1]
        data_time.update_state(time.time() - end)

        with tf.GradientTape() as tape:
            outputs = model(imgs, training=True)
            #print(outputs)

            if isinstance(outputs, tuple):
                loss = DeepSupervision(criterion, outputs, pids)
            else:
                loss = criterion(outputs, pids)
            #print("loss:",loss)
        grads = tape.gradient(loss, model.trainable_variables)
        optimizer.apply_gradients(
            zip(grads, model.trainable_variables))

        batch_time.update_state(time.time() - end)
        end = time.time()

        #print("total loss:",loss * pids.shape[0])
        if isinstance(outputs, tuple):
            targets = tf.tensor_scatter_nd_update(tf.zeros(outputs[0].shape), tf.transpose([tf.range(outputs[0].shape[0]), pids]), tf.ones(outputs[0].shape[0]))
            accuracy.update_state(targets, outputs[0])
        else:
            targets = tf.tensor_scatter_nd_update(tf.zeros(outputs.shape), tf.transpose([tf.range(outputs.shape[0]), pids]), tf.ones(outputs.shape[0]))
            accuracy.update_state(targets, outputs)
        losses.update_state(loss * pids.shape[0])
        temp_lr = optimizer.lr.numpy()

        if (batch_idx+1) % FLAGS.print_freq == 0:
            print('\tEpoch: [EP NO: {0}][BT NO: {1}]\t'
                  'LearningRate {temp_lr:.4f}\t'
                  'Time ({batch_time:.3f})\t'
                  'Data ({data_time:.3f})\t'
                  'Loss ({loss:.4f})\t'.format(
                   epoch+1, batch_idx+1, temp_lr=temp_lr, batch_time=batch_time.result().numpy(),
                   data_time=data_time.result().numpy(), loss=losses.result().numpy()))

        if batch_idx+1 == trian_max_batches:          #trian_max_batches
            break
    return losses, accuracy

def test(model, query_datagen, query_imgs, query_labels, gallery_datagen, gallery_imgs, gallery_labels, ranks=[1, 5, 10, 20]):
    """
    # creating batch_time with AverageMeter
    # make eval the model

    # with no_grad
        # create query_features, query_ids, query_camids
        # iterate as per query batches
            # convert images & set end time
            # make prediction and update batch_time

            # update query_features, query_ids, query_camids
        # convery query_features, query_ids, query_camids into array

        # create gallery_features, gallery_ids, gallery_camids
        # iterate as per gallery batches
            # convert images & set end time
            # make prediction and update batch_time

            # update gallery_features, gallery_ids, gallery_camids
        # convery gallery_features, gallery_ids, gallery_camids into array

    # creates m, n where m for total query_features & n for total gallery_features
    # calculate dismat
    # convert dismat to numpy array

    # calculate cmc, mAP from evaluate function
    # print out mAP and CMC curve

    Args:
        model ([type]): [description]
        query_datagen ([type]): [description]
        query_imgs ([type]): [description]
        query_labels ([type]): [description]
        gallery_datagen ([type]): [description]
        gallery_imgs ([type]): [description]
        gallery_labels ([type]): [description]
        ranks (list, optional): [description]. Defaults to [1, 5, 10, 20].
    """
    batch_time = tf.keras.metrics.Mean('test_batch_time', dtype=tf.float32)

    query_max_batches = query_imgs.shape[0] // FLAGS.test_batch if query_imgs.shape[0] % FLAGS.test_batch == 0 else (query_imgs.shape[0] // FLAGS.test_batch) + 1
    qf, q_pids, q_camids = [], [], []
    for batch_idx, (imgs, labels) in enumerate(query_datagen.flow(query_imgs, query_labels, batch_size=FLAGS.test_batch)):
        pids, camids = labels[:,0], labels[:,1]
        end = time.time()

        features = model(imgs)
        batch_time.update_state(time.time() - end)

        qf.append(features)
        q_pids.extend(pids)
        q_camids.extend(camids)

        if batch_idx + 1 == query_max_batches:
            break
    qf = tf.concat(qf, 0)
    q_pids = np.asarray(q_pids)
    q_camids = np.asarray(q_camids)

    print("\tExtracted features for query set, obtained {}-by-{} matrix".format(qf.shape[0], qf.shape[1]))

    gallery_max_batches = gallery_imgs.shape[0] // FLAGS.test_batch if gallery_imgs.shape[0] % FLAGS.test_batch == 0 else (gallery_imgs.shape[0] // FLAGS.test_batch) + 1
    gf, g_pids, g_camids = [], [], []
    for batch_idx, (imgs, labels) in enumerate(gallery_datagen.flow(gallery_imgs, gallery_labels, batch_size=FLAGS.test_batch)):
        pids, camids = labels[:,0], labels[:,1]
        end = time.time()

        features = model(imgs)
        batch_time.update_state(time.time() - end)

        gf.append(features)
        g_pids.extend(pids)
        g_camids.extend(camids)

        if batch_idx + 1 == gallery_max_batches:
            break
    gf = tf.concat(gf, 0)
    g_pids = np.asarray(g_pids)
    g_camids = np.asarray(g_camids)

    print("\tExtracted features for gallery set, obtained {}-by-{} matrix".format(gf.shape[0], gf.shape[1]))
    print("==> BatchTime(s)/BatchSize(img): {:.3f}/{}".format(batch_time.result().numpy(), FLAGS.test_batch))

    m, n = qf.shape[0], gf.shape[0]
    distmat = tf.broadcast_to(tf.reduce_sum(tf.pow(qf, 2), axis=1, keepdims=True), [m, n]) + \
              tf.transpose(tf.broadcast_to(tf.reduce_sum(tf.pow(gf, 2), axis=1, keepdims=True), [n, m]))
    distmat = distmat + (tf.matmul(qf, tf.transpose(gf)) * -2)
    distmat = distmat.numpy()

    print("Computing CMC and mAP")
    cmc, mAP = evaluate(distmat, q_pids, g_pids, q_camids, g_camids)

    print("Results ----------")
    print("\tmAP: {:.1%}".format(mAP))
    print("\tCMC curve")
    for r in ranks:
        print("\tRank-{:<3}: {:.1%}".format(r, cmc[r-1]))
    print("\t------------------")

    return cmc[0], mAP

if __name__ == "__main__":
    app.run(main)


#!/usr/bin/python3
# -*- coding: utf-8 -*-

def solve():
    j, p, s, k = map(int, input().split())
    res = []
    p1 = {}
    p2 = {}
    p3 = {}
    for x1 in range(1, j+1):
        for x2 in range(1, p+1):
            for x3 in range(1, s+1):
                d1 = (x1, x2)
                d2 = (x1, x3)
                d3 = (x2, x3)
                if p1.get(d1, 0) >= k or p2.get(d2, 0) >= k or p3.get(d3, 0) >= k:
                    continue
                p1[d1] = p1.get(d1, 0) + 1
                p2[d2] = p2.get(d2, 0) + 1
                p3[d3] = p3.get(d3, 0) + 1
                res.append('%d %d %d' % (x1, x2, x3))
    res2 = solve2(j, p, s, k)
    if len(res) != len(res2):
        print (j, p, s, k, res, res2)
    return '\n'.join([str(len(res))] + res)

p1 = [[0 for _ in range(11)] for _ in range(11)]
p2 = [[0 for _ in range(11)] for _ in range(11)]
p3 = [[0 for _ in range(11)] for _ in range(11)]
cur = []

def solve():
    j, p, s, k = map(int, input().split())
    res = solve2(j, p, s, k)
    return '\n'.join([str(len(res))] + ['%d %d %d' % x for x in res])

def solve2(j, p, s, k):
    global u, values, best
    values = [(x1, x2, x3) for x1 in range(1, j+1) for x2 in range(1, p+1) for x3 in range(1, s+1)]
    u = [False for _ in values]
    best = []
    dfs(k)
    return best

def dfs(k, i=0):
    if len(cur) > len(best):
        best[:] = cur
    for t in range(i, len(values)):
        x1, x2, x3 = values[t]
        if u[t] or p1[x1][x2] >= k or p2[x1][x3] >= k or p3[x2][x3] >= k:
            continue
        u[t] = True
        cur.append(values[t])
        p1[x1][x2] += 1
        p2[x1][x3] += 1
        p3[x2][x3] += 1
        dfs(k, t+1)
        p1[x1][x2] -= 1
        p2[x1][x3] -= 1
        p3[x2][x3] -= 1
        cur.pop()
        u[t] = False

if __name__ == '__main__':
    t = int(input())
    for i in range(t):
        print('Case #%d:' % (i+1), solve())


#definition of different custom loss functions
import tensorflow as tf

#to maximise activations
def activations(network, layers):
    loss = 0
    for i in network.keys():
        if i not in layers:
            try:
                for j in network[i].keys():
                    if j == 'concat':
                        loss += tf.log(tf.reduce_mean(tf.abs(network[i][j]))) #total blob activations
            except:
                loss += tf.log(tf.reduce_mean(tf.abs(network[i]))) #total blob activations
    return loss


import argparse
import logging
from core.pipelines.pipelines_controller import (
    get_pipeline_description,
    load_pipeline_dynamically,
)

plid = "bone_segmentation"

def get_description():
    return get_pipeline_description(plid)

def add_parser_arguments(parser):
    parser.add_argument(
        "input",
        metavar="i",
        type=str,
        help="Specify the path to the directory containing the scans in the form of DICOM",
    )
    parser.add_argument(
        "output",
        metavar="o",
        type=str,
        help="Specify the path of a single output file in the form of a glb file",
    )
    parser.add_argument(
        "-q",
        "--quiet",
        action="store_true",
        help="Set the logging level from INFO to ERROR",
    )
    parser.set_defaults(which=plid)

def run(args):
    logging.info("Loading and initializing bone pipeline dynamically")
    input_dir = args.input
    output_path = args.output
    pipeline_module = load_pipeline_dynamically(plid)
    pipeline_module.run(input_dir, output_path, [1])
    logging.info("Done.")

def main():
    parser = argparse.ArgumentParser(description=get_description())
    add_parser_arguments(parser)
    args = parser.parse_args()
    logging.basicConfig(
        level=logging.ERROR if args.quiet else logging.INFO,
        format="%(asctime)s - %(module)s:%(levelname)s - %(message)s",
        datefmt="%d-%b-%y %H:%M:%S",
    )

    run(args)

if __name__ == "__main__":
    main()


import uuid
from pydantic import BaseModel
from typing import List, Optional
from fastapi import Request

class CommodityBase(BaseModel):
    item: str
    itemCategory: str
    quantity: int
    price: int

class CommodityList(BaseModel):
    items: List[CommodityBase]

class ProductCreateForm:
    def __init__(self, request: Request):
        self.request: Request = request
        self.errors: List = []
        self.item: Optional[str] = None
        self.item_category: Optional[str] = None
        self.quantity: Optional[int] = None
        self.price: Optional[int] = None

    async def load_data(self):
        form = await self.request.form()
        self.item_id = str(uuid.uuid4)
        self.item = form.get("item")
        self.item_category = form.get("item_category")
        self.quantity = form.get("quantity")
        self.price = form.get("price")
        self.product = {
            'item': self.item,
            'item_category': self.item_category,
            'quantity': int(self.quantity),
            'price': int(self.price)
        }

    def is_valid(self):
        if not self.item:
            self.errors.append("A valid item is required")
        if not self.item_category:
            self.errors.append("Item Category is required")
        if not self.quantity:
            self.errors.append("Qunatity is required")
        if not self.price:
            self.errors.append("Price si required")
        if not self.errors:
            return True
        return False