import geopandas
import libpysal
import numpy
import pandas
import spopt
from spopt.region import RegionKMeansHeuristic
import warnings
import matplotlib.pyplot as plt
import geopandas as gpd

dim1 = 4
dim2 = 4
w = libpysal.weights.lat2W(dim1, dim2)
w.n
noclusters = 5
RANDOM_SEED = 12345
numpy.random.seed(RANDOM_SEED)
data = numpy.random.normal(size=(w.n, 3))
data.shape

# print(w.neighbors)

libpysal.weights.build_lattice_shapefile(dim1, dim2, "lattice.shp")
gdf = geopandas.read_file("lattice.shp")
gdf.plot(column="ID")

model = RegionKMeansHeuristic(data, noclusters, w)
model.solve()

print(model.labels_)

areas = numpy.arange(dim1 * dim2)
regions = [areas[model.labels_ == region] for region in range(5)]

# gdf["region"] = model.labels_
# gdf.plot(column="region")
labels = model.labels_
print(type(labels))
# print(regions)
# print(model.labels_)
# print(type(regions))
# print(model.centroids_)

# europe_land = gpd.read_file("C:/Users/defor/Desktop/Thesis/GEP/Clustering/Main Functions/europe.geojson")
# print(europe_land.geometry[0])
# print(type(europe_land))

current_amount_of_clusters = noclusters
off_shore = list()
for i in range(len(regions)):
    off_shore.append(list())
    for number in regions[i]:
        if number == 0 or number ==12 or number ==10:
            regions[i] = regions[i][regions[i] != number]
            off_shore[-1].append(number)
            labels[number] = current_amount_of_clusters

    if len(off_shore[-1])==0:
        off_shore = off_shore[:-1]
    else:
        current_amount_of_clusters += 1
print(labels)
# print(regions)
# print(off_shore)
# print(model.labels_)
# print(labels)
# plt.show()

# EEZ = geopandas.read_file('C:/Users/defor/Desktop/Thesis/GEP/Clustering/Main Functions/EEZ_Europe_land.geojson')

# dict_wind = dict()

# for country_name in EEZ["UNION"]:
#     dict_wind[country_name] = 0

# # print(dict_wind)

# i = 0
# for countryname in EEZ["UNION"]:
#     if countryname=="Estonia" or countryname=="Portugal":
#         dict_wind[EEZ["UNION"][i]] += 1/(i+1)
#     i += 1

# print(dict_wind)

# for cluster in regions_wind:
#     size_cluster = len(cluster)
#     for number in cluster:
#         i = 0
#         for polygon in EEZ["geometry"]:
#             if polygon.contains(coordinates[number]):
#                 dict_wind[EEZ["UNION"][i]] += 1/size_cluster
#             else:
#                 i += 1


s1 = "leetscode"
s2 = "lts"

def lcp_two_strings(s1, s2):

    n1 = len(s1)
    n2 = len(s2)

    lcp = ""

    for i in range(0,min(n1,n2)):
        if s1[i] == s2[i]:
            lcp = s1[0:i+1]
        else:
            return lcp
    return lcp

strings = ["flower","flow","flight"] # ans = fl
strings1 = ["dog","racecar","car"] # ans = ""
def longestCommonPrefix(strs):
        """
        :type strs: List[str]
        :rtype: str
        """
        if len(strs) == 0:
            return ""
        if len(strs) == 1:
            return strs[0]
        lcps = strs

        while(len(lcps) != 1):
            lcp = lcp_two_strings(lcps[0],lcps[1])
            lcps.append(lcp)
            del lcps[0:2]

        return lcps[0]

print(longestCommonPrefix(strings))
print(longestCommonPrefix(strings1))


#!/usr/bin/env python3

""" A simple parser for the Enron e-mail corpus """

import argparse
import json
import matplotlib.pyplot as plt
import operator
import dateutil.parser
import itertools
import pandas as pd
import numpy as np
import networkx as nx
from collections import Counter, defaultdict
import timeit

def process_arguments():
    """ Process command line arguments """
    parser = argparse.ArgumentParser(description="Enron Corpus Parser")
    parser.add_argument('-i', '--infile', type=argparse.FileType('r'), required=True,
                      help='Path to data file to process')
    parser.add_argument('-o', '--outfile', type=argparse.FileType('w'), required=True,
                      help='Path to output data to')
    return parser.parse_args()

def load_data(infile):
    """ Load enron data from file """
    data = json.load(infile)
    for datum in data:
      datum['timestamp'] = dateutil.parser.parse(datum['timestamp'])
    return sorted(data, key = operator.itemgetter('timestamp'))

def main():
    """ Application entry point """
    args = process_arguments()
    print('Loading data...')
    data = load_data(args.infile)
    people = list(set(m['sender'] for m in data) | set(r for m in data for r in m['recipients']))

    counts = defaultdict(Counter)
    for message in data:
        counts[message['sender']].update(message['recipients'])
    for person in people:
        counts[person][person]=0
    df = pd.DataFrame(counts)
    connections = (df * df.T).stack().to_frame()
    connections.index.names=['sender', 'recipient']
    connections.reset_index(inplace=True)
    subset = connections[connections['sender'] < connections['recipient']].copy()
    subset.columns = ['sender', 'recipient', 'product']
    subset['rank'] = subset['product'].rank(method='first', ascending=False)
    ranking = subset.sort_values(by='rank').set_index(['rank'])
    ranking.to_csv(args.outfile)

    gr = nx.Graph()
    for sender, recipient in zip(ranking['sender'], ranking['recipient']):
         gr.add_edge(sender, recipient)
    nx.write_dot(gr, 'thing.dot')

    args.infile.close()
    args.outfile.close()

if __name__ == '__main__':
    main()


import os
import subprocess
import hashlib
import openai
import json
from pathlib import Path

# receive email in O365 to check Apache AGE release
# Use Power Automate to send email to gpt-3 to extract commit hash and fingerprint
# clone and check out commit hash
# check fingerprint
# if fingerprint matches, then download and check the checksum
# if checksum matches, then build and test
# if test passes, then email success of checks

PG_VERSION = os.getenv("PG_VERSION") or "12"
AGE_VERSION = os.getenv("AGE_VERSION") or "1.3.0"
RC_VERSION = os.getenv("RC_VERSION") or "rc0"
SIG_FINGERPRINT = os.getenv("FINGERPRINT").replace(' ', '')
COMMIT_HASH = os.getenv("COMMIT_HASH")
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
APACHE_AGE_URL = f"https://dist.apache.org/repos/dist/dev/age/PG{PG_VERSION}/{AGE_VERSION}.{RC_VERSION}/"
GITHUB_AGE_URL = "https://github.com/apache/age.git"
AGE_DIRNAME = f"apache-age-{AGE_VERSION}"
AGE_PATH = Path(AGE_DIRNAME)
GIT_DIRNAME = f"{AGE_DIRNAME}-git"
GIT_PATH = Path(GIT_DIRNAME)
AGE_GZIP_FILENAME = f"apache-age-{AGE_VERSION}-src.tar.gz"
AGE_HASH_FILENAME = f"{AGE_GZIP_FILENAME}.sha512"
AGE_ASC_FILENAME = f"{AGE_GZIP_FILENAME}.asc"
SUCCESS_EMOJI="✅"
FAILURE_EMOJI="❌"
WORKING_EMOJI="💡"

# Function to print some of the global variables
def print_globals():
    print(f"PG_VERSION: {PG_VERSION}")
    print(f"AGE_VERSION: {AGE_VERSION}")
    print(f"RC_VERSION: {RC_VERSION}")
    print(f"SIG_FINGERPRINT: {SIG_FINGERPRINT}")
    print(f"COMMIT_HASH: {COMMIT_HASH}")
    print(f"OPENAI_API_KEY: sk-...{OPENAI_API_KEY[-3]}]")
    print(f"APACHE_AGE_URL: {APACHE_AGE_URL}")
    print(f"GITHUB_AGE_URL: {GITHUB_AGE_URL}")
    print(f"AGE_DIRNAME: {AGE_DIRNAME}")
    print(f"AGE_PATH: {AGE_PATH}")
    print(f"GIT_DIRNAME: {GIT_DIRNAME}")
    print(f"GIT_PATH: {GIT_PATH}")
    print(f"AGE_GZIP_FILENAME: {AGE_GZIP_FILENAME}")
    print(f"AGE_HASH_FILENAME: {AGE_HASH_FILENAME}")
    print(f"AGE_ASC_FILENAME: {AGE_ASC_FILENAME}")

def get_apache_release():
    # Download the gzip, SHA512 hash, and PGP signature files using curl
    for filename in (AGE_GZIP_FILENAME, AGE_HASH_FILENAME, AGE_ASC_FILENAME):
        download_command = f"curl -o {filename} {APACHE_AGE_URL}{filename}"
        subprocess.run(download_command, shell=True, check=True)

    # Decompress the gzip file
    decompress_command = f"tar xf {AGE_GZIP_FILENAME}"
    subprocess.run(decompress_command, shell=True, check=True)

# Function to verify the PGP signature of a file
def verify_pgp_signature(file_path):
    verify_command = f"gpg --verify {file_path}"
    output = subprocess.run(verify_command, shell=True, check=True, capture_output=True)

    openai.api_key = OPENAI_API_KEY

    prompt = f"Determine if this gpg output has a good signature. Return the response as json with the fields good_sig and fingerprint: {output.stderr.decode()}"
    print(f"Prompt: {prompt}")

    parameters = {
        "model": "gpt-3.5-turbo",
        "max_tokens": 250,
        "n": 1,
        "temperature": 0.5,
        "messages": [
            {
            "role": "system",
            "content": "You are a helpful assistant that is an expert in parsing gpg signatures and turning information into JSON objects. Your JSON should be lowercase, snakecase, and well-formed valid JSON. "
            },
            {
            "role": "user",
            "content": prompt
            }
        ]
    }
    response = openai.ChatCompletion.create(**parameters)
    text = response.choices[0].message.content
    json_response = json.loads(text)
    print(f"Response: {json_response}")
    if json_response["good_sig"] == "yes":
        print(f"{SUCCESS_EMOJI} PGP signature of {file_path} verified")
    if json_response["fingerprint"].replace(' ', '') == SIG_FINGERPRINT:
        print(f"{SUCCESS_EMOJI} PGP fingerprint of {file_path} verified")

# Function to verify the SHA512 hash of a file
def verify_sha512_hash(file_path):
    with open(file_path, 'r') as file:
        file_hash = file.read().split()[0]

    hasher = hashlib.sha512()
    with open(file_path[:-7], 'rb') as file:
        for chunk in iter(lambda: file.read(4096), b''):
            hasher.update(chunk)

    if file_hash == hasher.hexdigest():
        print(f"{SUCCESS_EMOJI} SHA512 hash of\n{file_hash}\n from {file_path} --> verified against {file_path[:-6]} to yield\n{hasher.hexdigest()}")
    else:
        print(f"{FAILURE_EMOJI} SHA512 hash of {file_path[:-7]} did not match")

# Function to clone a Git repository
def clone_repo(repo_url, target_directory, commit_hash=None):
    subprocess.run(["git", "clone", repo_url, target_directory])
    global GIT_PATH
    GIT_PATH = Path(GIT_DIRNAME)

    if commit_hash:
        # Change to the target directory and checkout the commit hash
        os.chdir(target_directory)
        subprocess.run(["git", "checkout", commit_hash])
        os.chdir("..")
        print(f"{SUCCESS_EMOJI} Git repo {repo_url} cloned to {target_directory} at commit {commit_hash}")
    else:
        print(f"{FAILURE_EMOJI} Git repo {repo_url} cloned to {target_directory} but no commit hash was provided")

# Function to check git tag
def check_git_tag(target_directory):
    # Change to the target directory and checkout the commit hash
    os.chdir(target_directory)
    # Check tag
    subprocess.run(["git", "fetch", "origin", "refs/tags/*:refs/tags/*"])
    tag = f"PG{PG_VERSION}/v{AGE_VERSION}-{RC_VERSION}"
    resp = subprocess.run(["git", "rev-list", "-n", "1", f"{tag}"], capture_output=True)
    tag_commit = resp.stdout.decode().strip()

    if tag_commit != COMMIT_HASH:
        print(f"{FAILURE_EMOJI} Error: Git tag commit {tag_commit} \ndoes not match commit hash {COMMIT_HASH}")
    else:
        print(f"{SUCCESS_EMOJI} Git tag {tag} verified")
    os.chdir("..")

# Function to calculate file checksum using SHA256
def calculate_checksum(file_path):
    hasher = hashlib.sha256()
    with open(file_path, 'rb') as file:
        for chunk in iter(lambda: file.read(4096), b''):
            hasher.update(chunk)
    return hasher.hexdigest()

def compare_checksums():
    # Compare the file checksums between the two clones
    match_count = 0
    mismatch_count = 0
    found_in_release_count = 0

    for file_a in AGE_PATH.glob("**/*"):
        if file_a.is_file():
            file_b = GIT_PATH / file_a.relative_to(AGE_PATH)

            if file_b.is_file():
                checksum_a = calculate_checksum(file_a)
                checksum_b = calculate_checksum(file_b)

                if checksum_a == checksum_b:
                    match_count += 1
                else:
                    print(f"{FAILURE_EMOJI} Mismatch: {file_a} and {file_b}")
                    mismatch_count += 1
            else:
                print(f"{FAILURE_EMOJI} File {file_b} not found in Git release, but in Apache release")

    print(f"Total matches: {match_count}")
    print(f"Total mismatches: {mismatch_count}")
    if mismatch_count == 0 and found_in_release_count == 0:
        print(f"{SUCCESS_EMOJI} All checksums and files match between Apache release and Git release")

# main
# print out the global vars calling function
print(f"{WORKING_EMOJI} Printing global variables...")
print_globals()
print(f"{WORKING_EMOJI} Getting Apache release...")
get_apache_release()
print(f"{WORKING_EMOJI} Verifying PGP signature...")
verify_pgp_signature(AGE_ASC_FILENAME)
print(f"{WORKING_EMOJI} Verifying SHA512 hash...")
verify_sha512_hash(AGE_HASH_FILENAME)
print(f"{WORKING_EMOJI} Cloning repo...")
clone_repo(GITHUB_AGE_URL, GIT_DIRNAME, COMMIT_HASH)
print(f"{WORKING_EMOJI} Checking git tag...")
check_git_tag(GIT_DIRNAME)
print(f"{WORKING_EMOJI} Comparing checksums...")
compare_checksums()


# Q.convert temperature c to f and write to temp.txt files
# f=c*9/5+32
temperatures = [10,-20,-289,100]

def c_to_f(temperatures):
    with open("temps.txt", 'w') as file:
        for c in temperatures:
            if c>-273.8:
                f=c*9/5+32
                file.write(str(f) + "\n")
c_to_f(temperatures)

06 - File Handling/3  - write_file_with_loop.py


""" Text-Classify by DeepLearning """

import pandas as pd
from tqdm import tqdm
from sklearn import preprocessing
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from src.utils.word2vec import sent2vec
from src.utils.matplot import draw_acc
import numpy as np
from keras.models import Sequential
from keras.utils import np_utils
from keras.layers.core import Dense, Activation, Dropout
from keras.layers.normalization import BatchNormalization
import gensim

def number_normalizer(tokens):
    """ 将所有数字标记映射为一个占位符（Placeholder）。
        对于许多实际应用场景来说，以数字开头的tokens不是很有用，但这样tokens的存在也有一定相关性。
        通过将所有数字都表示成同一个符号，可以达到降维的目的。
    """
    return ("#NUMBER" if token[0].isdigit() else token for token in tokens)

class NumberNormalizingVectorizer(TfidfVectorizer):
    def build_tokenizer(self):
        tokenize = super(NumberNormalizingVectorizer, self).build_tokenizer()
        return lambda doc: list(number_normalizer(tokenize(doc)))

def word_vec():
    stwlist = [line.strip() for line in open('../data/stop_words.txt', 'r', encoding='utf-8').readlines()]
    tfv = NumberNormalizingVectorizer(min_df=3,
                                      max_df=0.5,
                                      max_features=None,
                                      ngram_range=(1, 2),
                                      use_idf=True,
                                      smooth_idf=True,
                                      stop_words=stwlist)

    data = pd.read_csv('./tmp_files/sentence_label2.txt', header=0)
    # data = data.sample(frac=0.1)

    lbl_enc = preprocessing.LabelEncoder()
    y = lbl_enc.fit_transform(data['label'].values)

    xtrain, xvalid, ytrain, yvalid = train_test_split(data['sentence'].values, y,
                                                      stratify=y,
                                                      random_state=42,
                                                      test_size=0.1, shuffle=True)

    tfv.fit(list(xtrain) + list(xvalid))
    X = data["sentence"]
    X = [i.split() for i in X]

    model = gensim.models.Word2Vec(X, min_count=5, window=8, size=100)

    xtrain_w2v = [sent2vec(x, model) for x in tqdm(xtrain)]
    xvalid_w2v = [sent2vec(x, model) for x in tqdm(xvalid)]

    xtrain_w2v = np.array(xtrain_w2v)
    xvalid_w2v = np.array(xvalid_w2v)
    ytrain_np = np.array(ytrain)
    yvalid_np = np.array(yvalid)

    np.save('./tmp_files/xtrain_w2v.npy', xtrain_w2v)
    np.save('./tmp_files/xvalid_w2v.npy', xvalid_w2v)
    np.save('./tmp_files/ytrain_np.npy', ytrain_np)
    np.save('./tmp_files/yvalid_np.npy', yvalid_np)

    return xtrain_w2v, xvalid_w2v, ytrain, yvalid

def nn_model(x_train, x_valid, y_train, y_valid):
    # 在使用神经网络前，对数据进行缩放
    scl = preprocessing.StandardScaler()
    xtrain_w2v_scl = scl.fit_transform(x_train)
    xvalid_w2v_scl = scl.transform(x_valid)

    # 对标签进行binarize处理
    ytrain_enc = np_utils.to_categorical(y_train)
    yvalid_enc = np_utils.to_categorical(y_valid)

    # # 创建1个3层的序列神经网络（Sequential Neural Net）
    model = Sequential()

    # input_dim:指定输入尺寸   第一层是Dense层（全连接层）输入的是维度为1*100的列向量（input_dim=100）
    model.add(Dense(300, input_dim=100, activation='relu'))
    # Dropout层添加到模型的现有层和之前的输出层之间
    model.add(Dropout(0.2))
    model.add(BatchNormalization())

    model.add(Dense(300, activation='relu'))
    model.add(Dropout(0.3))
    model.add(BatchNormalization())

    model.add(Dense(4))
    model.add(Activation('softmax'))

    # 模型编译   损失函数选择的是交叉熵（loss=keras.losses.categorical_crossentropy）
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

    model_train = model.fit(xtrain_w2v_scl, y=ytrain_enc, batch_size=64,
              epochs=20, verbose=1,
              validation_data=(xvalid_w2v_scl, yvalid_enc))

    return model_train

if __name__ == '__main__':
    # xtrain_w2v, xvalid_w2v, ytrain, yvalid = word_vec()
    xtrain_w2v = np.load('./tmp_files/xtrain_w2v.npy')
    xvalid_w2v = np.load('./tmp_files/xvalid_w2v.npy')
    ytrain = np.load('./tmp_files/ytrain_np.npy')
    yvalid = np.load('./tmp_files/yvalid_np.npy')

    model_train = nn_model(xtrain_w2v, xvalid_w2v, ytrain, yvalid)
    draw_acc(model_train)


from email.mime import base
from lib2to3.pytree import convert
from os import write
from requests import request
import requests
from openpecha.core.ids import get_base_id,get_initial_pecha_id
from datetime import datetime
from openpecha.core.layer import Layer, LayerEnum
from openpecha.core.pecha import OpenPechaFS
from openpecha.core.metadata import InitialPechaMetadata,InitialCreationType
from bs4 import BeautifulSoup

from openpecha.core.annotation import AnnBase, Span
from uuid import uuid4
from pathlib import Path
from openpecha import github_utils,config
from zipfile import ZipFile
from pyewts import pyewts
import re
import logging

def create_opf(text,pecha_id):
    texts =text.splitlines()
    base_id = get_base_id()
    opf_path = f"{pecha_id}/{pecha_id}.opf"
    opf = OpenPechaFS(path =opf_path)
    bases = {f"{base_id}":text}
    layers = {f"{base_id}": {LayerEnum.segment: get_segment_layer(texts)}}
    meta = get_meta()
    opf.base = bases
    opf.save_base()
    opf.layers = layers
    opf.save_layers()
    opf._meta = meta
    opf.save_meta()

    return base_id

def get_meta():
    instance_meta = InitialPechaMetadata(
            id=pecha_id,
            initial_creation_type=InitialCreationType.input,
            source_metadata={
                "title":"༄། །བྱང་ཆུབ་སེམས་དཔའི་སྤྱོད་པ་ལ་འཇུག་པ་བཞུགས་སོ།",
                "language": "bo"
            })
    return instance_meta

def get_segment_layer(texts):
    segment_annotations = {}
    char_walker =0
    for base_text in texts:
        segment_annotation,char_walker = get_segment_annotation(char_walker,base_text)
        segment_annotations.update(segment_annotation)

    segment_layer = Layer(annotation_type= LayerEnum.segment,
    annotations=segment_annotations
    )
    return segment_layer

def get_segment_annotation(char_walker,base_text):

    segment_annotation = {
        uuid4().hex:AnnBase(span=Span(start=char_walker, end=char_walker + len(base_text)))
    }

    return (segment_annotation,len(base_text)+1+char_walker)

def main():
    global pecha_id
    pecha_id = get_initial_pecha_id()
    text = Path("./tib_multilingual.txt").read_text()
    create_opf(text,pecha_id)

if __name__ == "__main__":
    main()


import streamlit as st
from bmoderiva import lib
from dotenv import load_dotenv
import os

load_dotenv()

df = lib.get_data(os.getenv('REMOBS_TOKEN'))
df1 = lib.get_data_bmo(os.getenv('REMOBS_TOKEN'))
mapbox_token = os.getenv("MAPBOX_TOKEN")

st.write("# BMO-BR DERIVA")
st.write(f"## DADO TAG")
if df.empty:
    st.write('#### Não há dados da TAG')
else:
    st.write(f"#### {(df['date_time'].min())} até {(df['date_time'].max())}")
    st.write(f"#### Última posição: LAT {(df['lat'].iloc[-1])}, LON {(df['lon'].iloc[-1])}")
    df = lib.calculate_distance(df)
    lib.plot_map(df)

st.write(f"## DADO BMO")
if df.empty:
    st.write('#### Não há dados da antena da BMO')
else:
    st.write(f"#### {(df1['date_time'].min())} até {(df1['date_time'].max())}")
    st.write(f"#### Última posição: LAT {(df1['lat'].iloc[-1])}, LON {(df1['lon'].iloc[-1])}")
    df1 = lib.calculate_distance(df1)
    df2 = lib.df_time_interval(df1)
    #lib.plot_map(df1)
    #st.write("## Histórico de Tempo")
    lib.plot_map_time(df2, mapbox_token)


from collections import Counter
t = int(input())
for i in range(t):
    size = int(input())
    array = list(map(int, input().split()))
    if size == 1:
        print("-1")
    else:
        heights = sorted(Counter(array).items())
        for idx in heights:
            array.append(idx[1])
        for idx in array:
            print(idx)
    array.clear()


'''
 # @ Author: Zhi Wu
 # @ Create Time: 2022-08-09 12:36:35
 # @ Modified by: Zhi Wu
 # @ Modified time: 2022-08-09 12:37:06
 # @ Description: Install scripts.
 '''

import setuptools

with open("README.md", "r", encoding="utf-8") as fh:
    long_description = fh.read()

setuptools.setup(
    name='HIBERTools',
    version='2.3.2',
    author='Zhi Wu',
    author_email='wzwyyx@mail.ustc.edu.cn',
    description='Tools of HIBER Dataset',
    long_description=long_description,
    long_description_content_type="text/markdown",
    url='https://github.com/wuzhiwyyx/HIBER/tree/main/HIBERTools',
    project_urls = {
        "Bug Tracker": "https://github.com/wuzhiwyyx/HIBER/issues"
    },
    license='MIT',
    packages=['HIBERTools'],
    install_requires=['numpy', 'lmdb', 'tqdm', 'opencv-python', 'matplotlib'],
)


from CoupActions import *
from CoupDeck import GAMECARDS, CARD_AMBR, CARD_ASSN, CARD_CAPT, CARD_CONT, CARD_DUKE
from CoupHistory import *

class CoupPlayer(object):
    next_id = 100

    def __init__(self, name):
        self.name = name
        self.newGame()
        self.id = CoupPlayer.next_id
        CoupPlayer.next_id += 1

    def newGame(self):
        self.coins = 2
        self.cards = [-2, -2]
        self.numCards = 0
        self.isAlive = True
        self.isPlaying = False
        self.history = None
        self.game = None

    def getIsPlaying(self):
        return self.isPlaying

    def setIsPlaying(self, value):
        self.isPlaying = value

    def getCoins(self):
        return self.coins

    def die(self):
        self.isAlive = False

    def getCards(self):
        return self.cards

    # add CoupGame reference to player
    def attachGame(self, game):
        self.game = game

    # add CoupHistory reference to player
    def attachHistory(self, history):
        assert isinstance(history, CoupHistory)
        self.history = history

    # ask player which card to lose from their hand
    # returns card ID (not indice)
    def chooseLostCard(self):
        print("--- (" + self.name + ") Choose which cards to lose:")
        chosen = -1
        while (chosen == -1):
            for i in range(len(self.cards)):
                print("------ " + str(i) + ") " + GAMECARDS[self.cards[i]])
            user_input = input()
            try:
                user_input = int(user_input)
                assert (user_input < len(self.cards))
                assert (user_input >= 0)
                chosen = self.cards[user_input]
                return chosen

            except:
                print("------- Invalid input received.")

    def lose_card(self):
        lost = self.cards[0]
        if (self.numCards == 2):
            lost = self.chooseLostCard()

        self.numCards -= 1
        if(lost==self.cards[0]):
            self.cards[0] = -2
        else:
            self.cards[1] = -2

        #check if player is alive
        if(self.numCards <= 0):
            self.die()
        elif(self.cards[0] == -2):
            # make the "dead" card second in their hand
            self.cards[0], self.cards[1] = self.cards[1], self.cards[0]

        return lost

    def getStatusString(self):
        output = "------ Current hand: "
        output += self.getHandString()
        output += " and "
        output += str(self.coins)
        output += " coins"
        return output

    def getHandString(self):
        if ((self.isAlive == False) or (self.numCards == 0)):
            return "No cards"
        else:
            if self.cards[0] == -2:
                assert self.cards[1] == -2

            output = []
            for i in range(self.numCards):
                if (self.cards[i] >= 0):
                    output.append(GAMECARDS[self.cards[i]])
                else:
                    print("Card at index x is:", i, self.cards[i], self.cards)
            return str(output)

    def getPossibleActions(self):
        actions = []
        if self.coins < 3:
            actions = ([ACTION_EXC, ACTION_STL, ACTION_TAX, ACTION_INC, ACTION_FOR])
        elif self.coins < 7:
            actions = ([ACTION_ASS, ACTION_EXC, ACTION_STL, ACTION_TAX, ACTION_INC, ACTION_FOR])
        elif self.coins < 10:
            actions = ([ACTION_ASS, ACTION_EXC, ACTION_STL, ACTION_TAX, ACTION_INC, ACTION_FOR, ACTION_COU])
        else:
            actions = ([ACTION_COU])

        if (self.game.noSteal()):
            # if it is not possible to steal (i.e. all players have 0 coins)
            actions.remove(ACTION_STL)

        return actions

    def getPossibleTruthfulActions(self):
        actions = self.getPossibleActions()

        if (ACTION_EXC in actions) and (CARD_AMBR not in self.cards):
            actions.remove(ACTION_EXC)
        if (ACTION_ASS in actions) and (CARD_ASSN not in self.cards):
            actions.remove(ACTION_ASS)
        if (ACTION_STL in actions) and (CARD_CAPT not in self.cards):
            actions.remove(ACTION_STL)
        if (ACTION_TAX in actions) and (CARD_DUKE not in self.cards):
            actions.remove(ACTION_TAX)

        return actions

    def getAction(self, possibleActions) -> int:
        #how will we ensure player knows game state?

        while(True):
            print("--- Type H for help, C for my cards and coins")
            user_input = input("--- (" + self.name + ") Which action to play? ")
            if (user_input.lower() == "h") or (user_input.lower() == "help"):
                for i in possibleActions:
                    print("------ " + str(i) + " - " + actionToString[i])
                continue
            elif (user_input.lower() == "c") or ("coin" in user_input.lower()) or ("card" in user_input.lower()):
                print(self.getStatusString())
                continue

            try:
                action = int(user_input)
                assert(action in possibleActions)
                return action
            except:
                print("------- Invalid action input received.")

    # for actions with a target, return player ID to target
    def getTarget(self, possibleTargets) -> int:
        #how will we ensure player knows game state?

        while(True):
            user_input = input("--- Who would you like to target? ")
            try:
                player_id = int(user_input)
                assert player_id in possibleTargets
                return player_id
            except:
                print("------- Invalid target input received.")

    # ask player which card to block with, or don't block at all (-1)
    def getBlock(self, action) -> int:
        possibleBlocks = []

        prompt = "--- (" + self.name + ") You are the target of "

        if action == ACTION_ASS:
            possibleBlocks.append(CARD_CONT)

        elif action == ACTION_STL:
            possibleBlocks.append(CARD_AMBR)
            possibleBlocks.append(CARD_CAPT)

        elif action == ACTION_FOR:
            prompt = "--- (" + self.name + ") A player wants to use "
            possibleBlocks.append(CARD_DUKE)

        prompt += actionToString[action]
        prompt += ". Do you want to block this action?\n"
        prompt += "------ Type N for no block\n"
        prompt += "------ Type C to view my cards and coins\n"

        for card in possibleBlocks:
            prompt += "------ Type " + str(card) + " to block with " + GAMECARDS[card] + "\n"

        while(True):
            user_input = input(prompt)
            try:
                if (user_input.lower() == "n"):
                    user_input = -1
                elif ((user_input.lower() == "c") or (user_input.lower() == "n")):
                    print(self.getStatusString())
                    continue
                else:
                    user_input = int(user_input)
                assert (user_input in possibleBlocks) or (user_input == -1)
                break
            except:
                print("------- Invalid input received.")

        return user_input

    # asks user whether they want to challenge an action
    # returns True if they want to challenge, False if they do not
    def getChallenge(self, currentPlayerInfo, card, prob) -> bool:

            prompt = "--- (" + self.name + ") "
            prompt += currentPlayerInfo["name"]
            prompt += " claims to have "
            prompt += GAMECARDS[card]
            prompt += ", probability of truth = "
            prompt += str(prob)
            prompt += ". Challenge? (Y/N) "

            while(True):
                user_input = input(prompt)
                if (user_input.lower() == "n"):
                    return False
                elif (user_input.lower() == "y"):
                    return True
                elif ((user_input.lower() == "c") or (user_input.lower() == "n")):
                    print(self.getStatusString())
                    continue
                else:
                    print("------- Invalid input received.")

    # ask player which cards to keep out of possibleCards
    # returns array of Card IDs (not indices)
    def exchange(self, possibleCards):
        print("--- (" + self.name + ") Choose which cards to keep from the following:")
        chosenCards = []
        while (len(possibleCards) != 2):
            for i in range(len(possibleCards)):
                print("------ " + str(i) + ") " + GAMECARDS[possibleCards[i]])
            user_input = input()
            try:
                user_input = int(user_input)
                assert (user_input < len(possibleCards))
                assert (user_input >= 0)
                chosenCards.append(possibleCards.pop(user_input))

            except:
                print("------- Invalid input received.")
        return chosenCards

if __name__ == "__main__":
    player1 = CoupPlayer("Player 1");
    player1.cards[0] = 4
    player1.cards[1] = 2

    #action = player1.getAction()
    #print(action, actionToString[action])
    '''
    block = player1.getBlock(ACTION_ASS)
    print(block, GAMECARDS[block])

    block = player1.getBlock(ACTION_STL)
    print(block, GAMECARDS[block])

    block = player1.getBlock(ACTION_FOR)
    print(block, GAMECARDS[block])
    '''


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Sep 24 15:27:31 2018

@author: marcos
"""

import silabas as si
import recortar as co
import os, itertools, shutil
import numpy as np
import matplotlib.pyplot as plt
from skimage.filters import threshold_otsu

#Consigo todos los archivos
home = os.getcwd()
ubicacion = os.path.join(home, 'Motivos', 'Secuencias')
ubi_nuevos = os.path.join(ubicacion,'Nuevos')
ubi_viejos = os.path.join(ubicacion,'Server')
archivos = [os.path.join(ubi_nuevos, f) for f in os.listdir(ubi_nuevos)]
archivos.extend([os.path.join(ubi_viejos, f) for f in os.listdir(ubi_viejos)])
archivos.sort() #archivos ordenados alfabeticamente!!

#%% Leo los motivos

#cargo todas las sílabas
motivos = []
secuencias = []
for file in archivos:
    motivos.extend(si.procesar_archivo(file))

#arreglo formato, extraigo duraciones y notas de potenciales highnotes:

for gesto in motivos:
    gesto.hago_pendiente()

#después de haber agregado 'bajada' a la lista
duraciones = {c:[] for c in si.categorias}
for gesto in motivos:
    duraciones[gesto.categoria].append(gesto.duracion)

#%% Grafico histograma de duraciones

for k, cat in enumerate(si.categorias):
    ax = plt.subplot(3,3,k+1)
    ax.set_title('{} ({} totales)'.format(cat,len(duraciones[cat])))
    ax.hist(duraciones[cat],range = [0,0.12])
    ax.set_xlim([0,0.12])
    ax.set_ylabel('Ocurrencia')
    ax.set_xlabel('Duración [s]')
plt.tight_layout()

### SE VE SEPARACIÓN de dos tipos de silencio

#%% Separo en dos tipos de silencio:

threshold = threshold_otsu(np.array(duraciones['silencio']))

#Convierto los silencios cortos de categoría.
for gesto in motivos:
    gesto.separo_categoria(threshold, si.conversor['s'], 'sil_corto')

#Redefino duraciones:
duraciones = {c:[] for c in si.categorias}
for gesto in motivos:
    duraciones[gesto.categoria].append(gesto.duracion)

#%% Vuelvo a plotear

salteado = False
for k, cat in enumerate(si.categorias):
    if cat=='seno':
        salteado = True
        continue
    if salteado:
        k -= 1
    ax = plt.subplot(3,3,k+1)
    ax.set_title('{} ({} totales)'.format(cat,len(duraciones[cat])))
    ax.hist(duraciones[cat],range = [0,0.12])
    ax.set_xlim([0,0.12])
plt.tight_layout()

#%% Pendientes
#TODO:
    #continuidad en frecuencia (implementado en versión anterior)
    #parametro de las exp.
    #parametros de cosenos
    #percu/varias

#%% Recorto distitntos gestos

a_cortar = ('exp', 'percu', 'varias') #faltan senos

#hago las carpetas
cortadas = {tipo:os.path.join('Motivos', 'Recortados', tipo) for tipo in a_cortar}
for carpeta in cortadas.values():
    shutil.rmtree(carpeta, ignore_errors=True)
    os.makedirs(carpeta)

nombre_actual = ''

for gesto in motivos:

    if gesto.nuevo: #sólo los nuevos

        #abro la imagen correspondiente a un sonograma (una sola vez)
        if gesto.archivo != nombre_actual:
            nombre_actual = gesto.archivo
            #el sonograma original:
            file = nombre_actual.replace('Secuencias', 'Sonogramas').replace('.txt', '.png')
            im = co.abro_imagen(file)

        if gesto.categoria in a_cortar:

            ic = co.cortar(im, gesto.ubicacion, gesto.duracion, *gesto.data[:2])
            guardar_en = os.path.join(cortadas[gesto.categoria],
                                      os.path.basename(gesto.archivo).replace('.txt', ''))
            co.guardar(ic, guardar_en)

#%% Miro secuencias

#Una lista de 4 elementos donde cada una contiene todas las combinaciones
#posibles de 2, 3, 4 y 5 letras respectivas.

categorias = si.categorias
abecedario = [chr(c) for c in range(ord('a'),ord('z'))]
codigo = {cat:abc for cat, abc in zip(categorias, abecedario)}
inv_codigo = {abc:cat for cat, abc in codigo.items()}

secuencias = si.armar_secuencias(motivos, codigo)

o = []
for orden in range(4):
    o.append([''.join(l) for l in itertools.product(sorted(codigo.values()), repeat=orden+2)])

#%% Hallo secuencias

#paraacelerarlo podría usar que las secuencias de longitud n contienen
#las de longitud n-1 y que por ende podría reducir la búsqueda después
#de la primera pasada guardando los índices de dódne encontré patrones.

veces = [] #contiene la cantidad de veces que aparece cada transición
for orden in o:
    esta_vez = []
    for sec in orden:
        total = 0
        for canto in secuencias:
            total += si.VecesQueAparece(canto,sec)
        if total == 0: total = np.nan
        esta_vez.append(total)
    veces.append(esta_vez)

# los arrays oi tienen la frecuencia de ocurrencia de cada transición

#cada columna tiene la cantidad de veces que se pasa de un estado dado
#la fila indica a qué estado

#ejemplos (orden 1): [0,0] = a-->a; [0,3] = a-->d; [3,5] = d-->f
o1 = np.reshape(veces[0],[len(codigo),len(codigo)]).T

#ejemplos (orden 2): [0,0] = aa-->a; [0,3] = aa-->d; [3,5] = ad-->f
o2 = np.reshape(veces[1],[len(codigo)**2,len(codigo)]).T

#ejemplos (orden 3): [0,0] = aa-->aa; [0,3] = aa-->ad; [3,5] = ad-->af
o3 = np.reshape(veces[2],[len(codigo)**2,len(codigo)**2]).T

#%% Grafico las matrices

plt.matshow(o1)
plt.colorbar()
plt.xticks(range(len(codigo)),sorted(codigo.values()))
plt.yticks(range(len(codigo)),sorted(codigo.values()))

plt.matshow(o2)
plt.xticks(range(len(codigo)**2),[c[-1] for c in o[0]])
plt.yticks(range(len(codigo)),sorted(codigo.values()))
plt.colorbar()

plt.matshow(o3)
plt.colorbar()
plt.xticks(range(len(codigo)**2),[c[-1] for c in o[0]])
plt.yticks(range(len(codigo)**2),[c[-1] for c in o[0]])

#%% Hago las cosas bien normalizadas

# Normalizo por columna: probabilidad de transición A ESE ESTADO.
# Como la cantidad de eventos en cada columna es igual a la cantidad de
# que aparece el gesto de partida, normalizo por eso

#frecuencia de aparición de cada gesto
f0 = [len(duraciones[cat]) for cat in categorias]

#usando broadcasting sobre matrices cuadradas, toma el array a brocastear
#como fila y lo repite varias veces (que es lo que necesito). Por ejemplo:

# |1 2| * |1,2| = |1*1 2*2|
# |3 4|           |3*1 4*2|
n1 = o1 / f0

#%% Miro probabilidad del primer gesto y frecuencia por momento

#Calcula las cuentas de cada categoría para un histograma categórico
def Cuentas(datos,categs,diccionario):
#toma una lista conteniendo los gestos codificados
    out = np.zeros(len(categs))
    for dato in datos:
        ind = categs.index(diccionario[dato])
        out[ind] += 1

    return out

def ElHisto(lascuentas):
    plt.bar(range(len(lascuentas)),lascuentas,align='center')
    plt.xticks(range(len(lascuentas)),categorias,rotation=45)

#miro el primer gesto en cada uno
primeros = []
for sec in secuencias:
    primeros.append(sec[0])

cant_primeros = Cuentas(primeros,categorias,inv_codigo)
ElHisto(cant_primeros)
#plt.bar(range(len(categorias)),cant_primeros,align='center')
#plt.xticks(range(len(categorias)),categorias,rotation=45)

#frecuenca total de cada gesto:
cant_totales = np.zeros(len(categorias))
for sec in secuencias:
    cant_totales += Cuentas(sec,categorias,inv_codigo)

ElHisto(cant_totales)

cant_inicio = np.zeros(len(categorias))
cant_medio= np.zeros(len(categorias))
cant_final= np.zeros(len(categorias))

for sec in secuencias:
    l = len(sec)
    cant_inicio += Cuentas(sec[:l//3],categorias,inv_codigo)
    cant_medio += Cuentas(sec[l//3:l//3*2],categorias,inv_codigo)
    cant_final += Cuentas(sec[2*l//3:],categorias,inv_codigo)

plt.subplot(411)
ElHisto(cant_inicio/sum(cant_inicio))
plt.subplot(412)
ElHisto(cant_medio/sum(cant_medio))
plt.subplot(413)
ElHisto(cant_final/sum(cant_final))
plt.subplot(414)
ElHisto(cant_primeros)

ultimos = []
for sec in secuencias:
    ultimos.append(sec[-1])

cant_ultimos = Cuentas(ultimos,categorias,inv_codigo)
ElHisto(cant_ultimos)


import pika

connection = pika.BlockingConnection(pika.ConnectionParameters('localhost'))
channel = connection.channel()

channel.queue_declare(queue='celery')

def callback(ch, method, properties, body):
    print(" [x] Received %r" % body)

channel.basic_consume(callback,
                      queue='celery',
                      no_ack=False)

print(' [*] Waiting for messages. To exit press CTRL+C')
channel.start_consuming()


import numpy as np
import pandas as pd
from scipy.linalg import block_diag, norm
from pyFBS.utility import coh_frf

class VPT(object):
    """
    Virtual Point Transformation (VPT) - enables transformation of measured responses to a virtual DoFs.  Current
    implementation enables rigid interface deformation modes and all 6 DoFs (3 translations + 3 rotations), have to
    be included in the transformation.

    :param ch: A DataFrame containing information on channels (i.e. outputs)
    :type ch: pd.DataFrame
    :param refch: A DataFrame containing information on reference channels (i.e. inputs)
    :type refch: pd.DataFrame
    :param vp_ch: A DataFrame containing information on virtual point channels
    :type vp_ch: pd.DataFrame
    :param vp_refch: A DataFrame containing information on reference virtual point channels
    :type vp_refch: pd.DataFrame
    :param Wu: Displacement weigting matrix
    :type Wu: array(float), optional
    :param Wf: Force weighting matrix
    :type Wf: array(float), optional
    :param sort_matrix: Sort transformation matrixes
    :type sort_matrix: bool, optional
    """

    def __init__(self, ch, refch, vp_ch, vp_refch, Wu = None, Wf = None, sort_matrix = True):
        self.sort_matrix = sort_matrix

        # Load the physical input-output DoFs
        self.Channels = ch
        self.RefChannels = refch

        # Load virtual input-output DoFs
        self.Virtual_Channels = vp_ch
        self.Virtual_RefChannels = vp_refch

        # Load Weighting matrices if None, no weighting is applied in the transformation
        self.Wu_p = Wu
        self.Wf_p = Wf

        # Define the IDM_U and IDM_F matrix
        self.define_IDM_U()
        self.define_IDM_F()

    def define_IDM_U(self):
        """
        Calculates Ru, Tu and Fu matrices based on the supplied position and orientation of Channels and Virtual
        Channels.
        """
        ov_u, _vps, mask_u = self.find_overlap_ch(self.Channels, self.Virtual_Channels)

        R_all = []

        # iterates through all unique virtual points (through grouping)
        _Warray = []
        for i in range(len(ov_u)):
            # gets the unique VP position
            _posVP = np.asarray(self.Virtual_Channels.iloc[i][["Position_1","Position_2","Position_3"]].to_numpy())
            # gets the unique VP orientation
            _dirVP = np.asarray(self.Virtual_Channels.iloc[i:i+3][["Direction_1","Direction_2","Direction_3"]].to_numpy())
            # gets the current positions
            ov_c = ov_u[i]
            # gets defined DoF for specific VP
            _desc = self.Virtual_Channels["Description"].to_list()

            r = np.zeros((len(ov_c[0]), len(_desc)))
            for j, ch in enumerate(ov_c[0]):
                _pos = np.asarray(self.Channels.iloc[ch][["Position_1","Position_2","Position_3"]].to_numpy()).astype(float)
                _dir = np.asarray(self.Channels.iloc[ch][["Direction_1","Direction_2","Direction_3"]].to_numpy()).astype(float)
                _group = self.Channels.iloc[ch]["Grouping"]
                _type = self.Channels.iloc[ch]["Quantity"]
                r[j, :] = (_dirVP @ _dir) @ self.R_matrix_U(_posVP - _pos, _desc, type=_type)
                _Warray.append(self.W_rotational(_pos, _dir, type=_type))
            R_all.append(r)

        Ru = block_diag(*R_all, np.eye(len(np.where(mask_u != 0)[0])))

        if self.sort_matrix:
            R_n = np.zeros_like(Ru)
            # position the transformation matrix based on location it the .xlsx file
            R_all = np.asarray(R_all)[0, :, :]
            gg = 0
            trig = True
            for i, k in enumerate(mask_u):
                if k == 1:
                    R_n[i, gg] = 1
                    gg += k
                else:
                    if trig:
                        R_n[i:i + R_all.shape[0], gg:gg + R_all.shape[1]] = R_all
                        gg += R_all.shape[1]
                        trig = False
            Ru = R_n

        # definition of weighting matrix
        if self.Wu_p == None:
            Wu = block_diag(*_Warray)
            Wu = block_diag(Wu, np.eye(len(np.where(mask_u != 0)[0])))
        else:
            Wu = block_diag(*self.Wu_p)
            Wu = block_diag(Wu, np.eye(len(np.where(mask_u != 0)[0])))

        # calculate the Tu, Fu matrices
        Tu = np.linalg.pinv(Ru.T @ Wu @ Ru) @ Ru.T @ Wu
        Fu = Ru @ Tu

        self.Ru = Ru
        self.Wu = Wu
        self.Tu = Tu
        self.Fu = Fu

    def define_IDM_F(self):
        """
        Calculates the Rf, Tf, Ff matrices based on the supplied position and orientation of Reference Channels and
        Reference Virtual Channels.
        """

        ov_f, _vps, mask_f = self.find_overlap_ch(self.RefChannels, self.Virtual_RefChannels)
        # print(mask_f)
        R_all = []
        # iterates through all unique virtual points (through grouping)
        for i in range(len(ov_f)):
            # gets the unique VP position
            _posVP = np.asarray(self.Virtual_RefChannels.iloc[i][["Position_1","Position_2","Position_3"]].to_numpy())
            # gets the unique VP orientation
            _dirVP = np.asarray(self.Virtual_RefChannels.iloc[i:i+3][["Direction_1","Direction_2","Direction_3"]].to_numpy())
            # gets the current positions
            ov_c = ov_f[i]
            # gets defined DoF for specific VP
            _desc = self.Virtual_RefChannels["Description"].to_list()

            r = np.zeros((len(ov_c[0]), len(_desc)))
            for j, ch in enumerate(ov_c[0]):
                _pos = np.asarray(self.RefChannels.iloc[ch][["Position_1", "Position_2", "Position_3"]].to_numpy()).astype(float)
                _dir = np.asarray(self.RefChannels.iloc[ch][["Direction_1", "Direction_2", "Direction_3"]].to_numpy()).astype(float)
                _group = self.RefChannels.iloc[ch]["Grouping"]
                _type = self.RefChannels.iloc[ch]["Quantity"]
                r[j, :] = (self.R_matrix_F(_posVP - _pos, _desc) @ (_dirVP @ _dir).T).reshape(-1)
            R_all.append(r)

        # position the transformation matrix based on location it the .xlsx file
        Rf = block_diag(*R_all, np.eye(len(np.where(mask_f != 0)[0])))

        if self.sort_matrix:
            R_n = np.zeros_like(Rf)
            R_all = np.asarray(R_all)[0, :, :]
            gg = 0
            trig = True
            for i, k in enumerate(mask_f):
                if k == 1:
                    R_n[i, gg] = 1
                    gg += k
                else:
                    if trig:
                        R_n[i:i + R_all.shape[0], gg:gg + R_all.shape[1]] = R_all

                        gg += R_all.shape[1]
                        trig = False
            Rf = R_n

        # definition of weighting matrix
        if self.Wf_p == None:
            Wf = np.eye(np.max(Rf.shape))
        else:
            Wf = self.Wf_p

        # calculate the Tf, Ff matrices
        Tf = Wf @ Rf @ np.linalg.pinv(Rf.T @ Wf @ Rf)
        Ff = Rf @ Tf.T

        self.Rf = Rf
        self.Wf = Wf
        self.Tf = Tf
        self.Ff = Ff

    @staticmethod
    def R_matrix_U(pos, desc, type="Acceleration"):
        """
        Calculate Ru matrix based on the channel position/orientation and sensor type.

        :param pos: Position of the channel.
        :type pos: array(float)
        :param type: Type of the channel (i.e. Acceleration or Angular Acceleration).
        :type pos: string, optional
        :returns: Ru matrix
        """

        rx, ry, rz = pos[0], pos[1], pos[2]

        _R = np.asarray([[1, 0, 0, 0, rz, -ry],
                         [0, 1, 0, -rz, 0, rx],
                         [0, 0, 1, ry, -rx, 0]])

        if type == "Angular Acceleration":
            _R = np.asarray([[0, 0, 0, 1, 0, 0],
                             [0, 0, 0, 0, 1, 0],
                             [0, 0, 0, 0, 0, 1]])

        # isolating desired DoF
        _R = np.asarray(pd.DataFrame(_R, columns=['ux','uy','uz','tx','ty','tz'])[desc])

        return _R

    @staticmethod
    def W_rotational(pos, dir, type="Angular Acceleration"):
        """
        Defines the weighting matrix based on the location of rotational accelerometer

        :param pos: Position of the channel.
        :type pos: array(float)
        :param dir: Direction of the channel
        :type dir: array(float)
        :param type: Type of the channel (i.e. Acceleration or Angular Acceleration)
        :type type: str, optional
        """

        rx, ry, rz = pos[0], pos[1], pos[2]

        _W = 1

        if type == "Angular Acceleration":
            c = np.where(np.asarray(dir) != 0)[1][0]
            if c == 0:
                _W = np.sqrt(rz ** 2 + ry ** 2) ** 2
            elif c == 1:
                _W = np.sqrt(rz ** 2 + rx ** 2) ** 2
            elif c == 2:
                _W = np.sqrt(ry ** 2 + rx ** 2) ** 2

        return _W

    @staticmethod
    def R_matrix_F(pos, desc):
        """
        Calculates Rf matrix based on the reference channel position/orientation.

        :param pos: Position of the reference channel relative to the virtual point.
        :type pos: array(float)
        :returns: Rf matrix
        """

        rx, ry, rz = pos[0], pos[1], pos[2]

        _R = np.asarray([[1, 0, 0],
                         [0, 1, 0],
                         [0, 0, 1],
                         [0, -rz, ry],
                         [rz, 0, -rx],
                         [-ry, rx, 0]])

        # isolating desired DoF
        _R = np.asarray(pd.DataFrame(_R.T, columns=['fx','fy','fz','mx','my','mz'])[desc]).T

        return _R

    @staticmethod
    def find_overlap_ch(channelsA, channelsB):
        """
        Finds an overlap of grouping number between two channel DataFrames.

        :param channelsA: First set of channels
        :type channelsA: pd.DataFrame
        :param channelsB: Second set of channels
        :type channelsB: pd.DataFrame
        :return: overlap,unique_index, overlap_mask
        """

        # Get the grouping numbers from DataFrames
        _group_ch = channelsA.Grouping.to_numpy()
        _group_chVP = channelsB.Grouping.to_numpy()

        # Find overlap between the two channel datasets
        _overlap = []
        for a in np.unique(_group_chVP):
            _overlap.append(np.where(_group_ch == a))

        mask = np.isin(_group_ch, np.unique(_group_chVP), invert=True).astype(int)

        return _overlap, np.unique(_group_chVP, return_index=True), mask

    @staticmethod
    def find_group(gr, gr_list):
        """
        Get a grouping overlap between two DataFrames.

        :param gr: Grouping number
        :type gr: int
        :param gr_list: A list of grouping numbers
        :type gr_list: list
        :return: overlap_mask
        """

        _overlap = []
        for a in np.unique(gr):
            _overlap.append(np.where(gr_list == a))
        return np.asarray(_overlap).reshape(-1)

    def apply_VPT(self, freq,FRF):
        """
        Applies the Virtual Point Transformation on the FRF matrix.

        :param freq: Frequency vector
        :type freq: array(float)
        :param FRF: A matrix of Frequency Response Functions FRFs [f,out,in].
        :type FRF: array(float)
        """

        _Y_vpt = self.Tu @ FRF @ self.Tf

        self.vptData = _Y_vpt
        self.freq = freq
        self.FRF = FRF

    def consistency(self, grouping, ref_grouping):
        """
        Evaluates VP consistency indicators based on the supplied grouping numbers.

        :param grouping: Grouping number of the VP.
        :type grouping: float
        :param ref_grouping: Grouping number of the reference VP.
        :type ref_grouping: float
        """

        # get all groupings from the vpt
        _ch_all = self.Channels.Grouping.to_numpy()
        _chVP_all = self.Virtual_Channels.Grouping.to_numpy()

        _Rch_all = self.RefChannels.Grouping.to_numpy()
        _RchVP_all = self.Virtual_RefChannels.Grouping.to_numpy()

        # extract the grouping mask
        ind_ch = self.find_group(grouping, _ch_all)
        ind_Rch = self.find_group(ref_grouping, _Rch_all)

        # Calculate sensor consistency
        sub_Y = np.transpose(self.FRF,(1,2,0))[ind_ch, :, :][:, ind_Rch, :]
        sub_Fu = self.Fu[ind_ch, :][:, ind_ch]

        u_f = np.zeros((sub_Y.shape[0], 1, sub_Y.shape[2]), dtype=complex)
        u = np.zeros((sub_Y.shape[0], 1, sub_Y.shape[2]), dtype=complex)

        for i in range(sub_Y.shape[2]):
            # filtered response
            u_f[:, :, i] = sub_Fu @ sub_Y[:, :, i] @ np.ones((sub_Y.shape[1], 1))
            # initial response
            u[:, :, i] = sub_Y[:, :, i] @ np.ones((sub_Y.shape[1], 1))

        self.u_f = u_f[:,0,:]
        self.u = u[:,0,:]

        # Calculate overall sensor consistency indicator
        self.overall_sensor = norm(self.u_f,axis = 0) / norm(self.u,axis = 0)

        # Calculate specific sensor consistency indicator
        specific_sensor = []
        for i in range(self.u.shape[0]):
            specific_sensor.append(coh_frf(self.u_f[i, :], self.u[i, :]))

        self.specific_sensor = np.asarray(specific_sensor)

        # Calculate impact consistency
        sub_Y = np.transpose(self.FRF,(1,2,0))[ind_ch, :, :][:, ind_Rch, :]
        sub_Ff = self.Ff[ind_Rch, :][:, ind_Rch]

        y_f = np.zeros((sub_Y.shape[1], 1, sub_Y.shape[2]), dtype=complex)
        y = np.zeros((sub_Y.shape[1], 1, sub_Y.shape[2]), dtype=complex)

        for i in range(sub_Y.shape[2]):
            # filtered response
            y_f[:, :, i] = (np.ones((sub_Y.shape[0], 1)).T @ sub_Y[:, :, i] @ sub_Ff).T
            # initial response
            y[:, :, i] = (np.ones((sub_Y.shape[0], 1)).T @ sub_Y[:, :, i]).T

        self.y_f = y_f[:,0,:]
        self.y = y[:,0,:]

        # Calculate overall impact consistency indicator
        self.overall_impact = norm(self.y_f,axis = 0) / norm(self.y,axis = 0)

        # Calculate specific impact consistency indicator
        specific_impact = []
        for i in range(self.y.shape[0]):
            specific_impact.append(coh_frf(self.y_f[i,:],self.y[i,:]))

        self.specific_impact = np.asarray(specific_impact)

    """
    Frequency-dependend weighting matrix - to be implemented in the pyFBS with next release

    Wu = block_diag(*_Warray)
    Wu = block_diag(Wu, np.eye(len(np.where(mask_u != 0)[0])))
    self.Wu = Wu

    # Wu_f is a 4D numpy array for each input set you use where -j refers to the freq input W[i,:,:,j]
    # n_imp... number of impacts
    # n_out... number of outputs
    # n_freq.. number of freqs

    n_out = self.Y.Data.shape[0]
    n_imp = self.Y.Data.shape[1]
    n_freq = self.Y.Data.shape[2]

    Wu_f = np.zeros((n_imp, n_out, n_freq))
    Tu_f = np.zeros((n_imp, self.Ru.shape[1], n_out, n_freq))
    # Fu_f = np.zeros((n_imp, n_out , n_out , 1000))

    for _f in tqdm(range(n_freq)):
        for _i in range(n_imp):
            _tW = block_diag(*np.abs(self.Y.Coherence[:, _i, _f])) ** 2
            # print(_tW.shape)
            # for _d,diag_val in enumerate(np.diag(_tW)):
            #    if _d in [9,10,11,21,22,23,33,34,35]:
            #        _tW[_d,_d] = 0#sigmoid(self.Y.Freqs[_f])

            W_sum = Wu + _tW
            Wu_f[_i, :, _f] = np.diag(W_sum)

            Tu = np.linalg.pinv(Ru.T @ W_sum @ Ru) @ Ru.T @ W_sum
            # Fu = Ru @ Tu
            Tu_f[_i, :, :, _f] = Tu
            # Fu_f[_i, :, :, _f] = Fu

    self.Tu = Tu
    self.Wu_f = Wu_f

    self.Tu_f = Tu_f
    self.Fu_f = Fu_f
    """


class Solution(object):
    def XXX(self, nums):
        """
        :type nums: List[int]
        :rtype: List[List[int]]
        """
        if not nums:
            return
        res = []
        n = len(nums)
        visited = [0] * n
        def helper1(temp_list,length):
            if length == n:
                res.append(temp_list)
            for i in range(n):
                if visited[i] :
                    continue
                visited[i] = 1
                helper1(temp_list+[nums[i]],length+1)
                visited[i] = 0
        def helper2(nums,temp_list,length):
            if length == n:
                res.append(temp_list)
            for i in range(len(nums)):
                helper2(nums[:i]+nums[i+1:],temp_list+[nums[i]],length+1)
        helper1([],0)
        return res


from __future__ import absolute_import, division, print_function, unicode_literals
import argparse
from scipy import inner, outer
import tensorflow as tf
import numpy as np
import pandas as pd
import matplotlib
matplotlib.use('Agg')
from sklearn.utils import shuffle
from sklearn import datasets, metrics
from sklearn.model_selection import train_test_split, KFold, StratifiedKFold
from random import seed
import time
import sys
import os
os.environ["CUDA_VISIBLE_DEVICES"]="3"

config = tf.ConfigProto()
config.gpu_options.allow_growth = True
#session = tf.Session(config=config)
#K.set_session(session)

print(tf.__version__)

parser = argparse.ArgumentParser(description='cv classification model')
parser.add_argument('-i', '--inpath', type=str, default='../TARGET_Fusion/', help='input file')
parser.add_argument('-o', '--outpath', type=str, default='../final_result/TARGET/methy+counts/', help='output file')
parser.add_argument('-d', '--dataset', type=str, default='TARGET', help='SEQC or TARGET')
parser.add_argument('-fus', '--fusion', type=str, default='net', help='fusion technique: none, net or feature')
# parser.add_argument('--rfe', type=str, default='none', help='RFE choice: none, svm, dt, rf, lr')
parser.add_argument('-fea', '--feature_type', type = str, default='both', help='both, cen or mod')
parser.add_argument('-m', '--mycn', action='store_true', default=False, help='add this tag to include mych feature')

args = parser.parse_args()
inpath = args.inpath
outpath = args.outpath
dataset = args.dataset
fusion = args.fusion
feature_type = args.feature_type
mycn = args.mycn

os.makedirs(outpath, exist_ok=True)

if mycn:
    preflix = 'mycn_'
else:
    preflix = ''

if fusion == 'net' or fusion == 'none':
    filename = 'new_scaled_result.csv'
elif fusion == 'feature':
    filename = 'new_scaled_result_FeaFusion.csv'
else:
    print('fusion tech error')

file1 = inpath + preflix + filename

expression = np.loadtxt(file1, dtype=float, delimiter = ",")
label_vec = np.array(expression[:,-1],dtype=int)

if feature_type == 'both':
    expression = np.array(expression[:, :-1])	# include both centrality and modularity features
elif feature_type == 'cen':
    if mycn:
        expression = np.append(expression[:, :13], np.expand_dims(expression[:, -2], axis=1), axis=1)
    else:
        expression = np.array(expression[:, :13])	# include only centrality features from GSE49710/62564
elif feature_type == 'mod':
    expression = np.array(expression[:, 13:-1])	# include only modularity features from GSE49710/62564
else:
    print('feature type error')

labels = []
for l in label_vec:
    if l==1:
        labels.append([0,1])    # 105 samples
    else:
        labels.append([1,0])    # 393 samples
labels = np.array(labels, dtype=int)
# labels = np.array(label_vec, dtype=int)
print('input features shape:', expression.shape)
print('labels shape:', labels.shape)

outer_cv = StratifiedKFold(n_splits=3, shuffle=True, random_state=40)
inner_cv = StratifiedKFold(n_splits=3, shuffle=True, random_state=40)

L2 = True
max_pooling = False
droph1 = False
display_step = 1

def dfn(x, layers, weights, biases, keep_prob):
    layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])
    layer_1 = tf.nn.relu(layer_1)
    dfn_layer = []
    dfn_layer.append(layer_1)
    if droph1:
        layer_1 = tf.nn.dropout(layer_1, keep_prob=keep_prob)
    if layers[0]:
        layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])
        layer_2 = tf.nn.relu(layer_2)
        layer_2 = tf.nn.dropout(layer_2, keep_prob = keep_prob)
        dfn_layer.append(layer_2)
    if layers[1]:
        layer_3 = tf.add(tf.matmul(layer_2, weights['h3']), biases['b3'])
        layer_3 = tf.nn.relu(layer_3)
        layer_3 = tf.nn.dropout(layer_3, keep_prob = keep_prob)
        dfn_layer.append(layer_3)
    if layers[2]:
        layer_4 = tf.add(tf.matmul(layer_3, weights['h4']), biases['b4'])
        layer_4 = tf.nn.relu(layer_4)
        layer_4 = tf.nn.dropout(layer_4, keep_prob=keep_prob)
        dfn_layer.append(layer_4)
    if layers[3]:
        layer_5 = tf.add(tf.matmul(layer_4, weights['h5']), biases['b5'])
        layer_5 = tf.nn.relu(layer_5)
        layer_5 = tf.nn.dropout(layer_5, keep_prob=keep_prob)
        dfn_layer.append(layer_5)

    out_layer = tf.matmul(dfn_layer[number_of_layers], weights['out']) + biases['out']

    return out_layer

# file = open("final_result/SEQC/Feature_level_fusion/feafusion_result_log", "w")

# all_layers = [[True, False, False, False],[True, False, False, False],
#               [True, True, False, False],[True, True, False, False],
#               [True, True, True, False],[True, True, True, False],[True, True, True, False],[True, True, True, False],[True, True, True, False],[True, True, True, False],
#               [True, True, True, True],[True, True, True, True],[True, True, True, True],[True, True, True, True],[True, True, True, True],[True, True, True, True],[True, True, True, True],[True, True, True, True]
# ]
# all_layers_size = [[4], [8],
#                    [8,8], [4,4],
#                    [8,64,16], [8,16,4], [8,4,2], [4,4,2], [4,2,2], [2,2,2],
#                    [8,64,16,8], [8,16,4,8], [8,8,4,4], [8,4,4,2], [8,2,2,2], [4,4,2,2], [4,2,2,2], [2,2,2,2]
#                    ]
all_layers = [[True, False, False, False],[True, False, False, False],
              [True, True, False, False],[True, True, False, False],
              [True, True, True, False],[True, True, True, False],[True, True, True, False],[True, True, True, False],
              [True, True, True, True],[True, True, True, True],[True, True, True, True],[True, True, True, True]
]
all_layers_size = [[4], [8],
                   [8,8], [4,4],
                   [8,16,4], [8,4,2], [4,4,2], [4,2,2],
                   [8,16,4,8], [8,8,4,4], [8,4,4,2], [4,4,2,2]
                   ]
all_learning_rates = [ 1e-2, 1e-3, 1e-4]
# all_learning_rates = [1e-3, 1e-4]
all_batch_sizes = [8,32]
# all_training_epochs = [100,1000]
# all_training_epochs = [1000]
training_epochs = 1000
count = 0
all_exp_count = (len(all_layers_size))*len(all_learning_rates)*len(all_batch_sizes)

log_outer=[]
k_outer = 0

# metrics for outer_cv
acc_outer, auc_outer, f1_outer = [], [], []
test_acc_outer, test_auc_outer, test_f1_outer = [], [], []

# outer_cv is for model selection (tune the parameters of the model)
for train_index, test_index in outer_cv.split(expression, label_vec):
    k_outer += 1
    # training set for outer_cv is the whole dataset for inner_cv
    inner_data = expression[train_index, :]
    inner_label = labels[train_index, :]

    # initialize the best hyperparameters
    log_inner=[]
    best_layer_size = all_layers_size[0]
    best_lr = all_learning_rates[0]
    best_bs = all_batch_sizes[0]

    for i in range(len(all_layers_size)):
        for learning_rate in all_learning_rates:
            # for training_epochs in all_training_epochs:
            for batch_size in all_batch_sizes:
                layers = all_layers[i]
                layers_size = all_layers_size[i]

                number_of_layers = 0
                for j in range(len(layers)):
                    if layers[j] is True:
                        number_of_layers = j+1

                n_hidden_1 = np.shape(expression)[1]
                n_classes = 2
                n_features = np.shape(expression)[1]

                x = tf.placeholder(tf.float32, [None, n_features])
                y = tf.placeholder(tf.float32, [None, n_classes])

                keep_prob = tf.placeholder(tf.float32)
                lr = tf.placeholder(tf.float32)

                weights = {
                    'h1': tf.Variable(tf.truncated_normal(shape=[n_features, n_hidden_1], stddev=1/np.sqrt(n_features)))
                }

                biases = {
                    'b1': tf.Variable(tf.zeros([n_hidden_1])),
                    'out': tf.Variable(tf.zeros([n_classes]))
                }
                if layers[0]:
                    weights['h2'] = tf.Variable(tf.truncated_normal(shape=[n_hidden_1, layers_size[0]], stddev=1/np.sqrt(n_hidden_1)))
                    biases['b2'] = tf.Variable(tf.zeros([layers_size[0]]))
                if layers[1]:
                    weights['h3'] = tf.Variable(tf.truncated_normal(shape=[layers_size[0], layers_size[1]], stddev=1/np.sqrt(layers_size[0])))
                    biases['b3'] = tf.Variable(tf.zeros([layers_size[1]]))
                if layers[2]:
                    weights['h4'] = tf.Variable(tf.truncated_normal(shape=[layers_size[1], layers_size[2]], stddev=1 / np.sqrt(layers_size[1])))
                    biases['b4'] = tf.Variable(tf.zeros([layers_size[2]]))
                if layers[3]:
                    weights['h5'] = tf.Variable(tf.truncated_normal(shape=[layers_size[2], layers_size[3]], stddev=1 / np.sqrt(layers_size[2])))
                    biases['b5'] = tf.Variable(tf.zeros([layers_size[3]]))

                weights['out'] = tf.Variable(tf.truncated_normal(shape=[layers_size[number_of_layers-1], n_classes], stddev=1/np.sqrt(layers_size[number_of_layers-1])))

                pred_inner = dfn(x, layers, weights, biases, keep_prob)

                # cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=pred, labels=y))
                if dataset == 'SEQC':
                    class_weights = tf.constant([1, 3.74])
                elif dataset == 'TARGET':
                    class_weights = tf.constant([1, 1.04])
                else:
                    print('wrong dataset type!')
                    exit()
                cost = tf.reduce_mean(tf.nn.weighted_cross_entropy_with_logits(logits=pred_inner, targets=y, pos_weight=class_weights))
                if L2:
                    reg = tf.nn.l2_loss(weights['h1'])
                    if layers[0]:
                        reg+=tf.nn.l2_loss(weights['h2'])
                    if layers[1]:
                        reg+=tf.nn.l2_loss(weights['h3'])
                    if layers[2]:
                        reg+=tf.nn.l2_loss(weights['h4'])
                    if layers[3]:
                        reg+=tf.nn.l2_loss(weights['h5'])
                    reg+=tf.nn.l2_loss(weights['out'])
                    cost = tf.reduce_mean(cost+0.001*reg)
                optimizer = tf.train.AdamOptimizer(learning_rate = lr).minimize(cost)

                correct_prediction = tf.equal(tf.argmax(pred_inner,1), tf.argmax(y,1))
                accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
                y_score = tf.nn.softmax(logits=pred_inner)

                loss_rec = np.zeros([training_epochs, 1])
                training_eval = np.zeros([training_epochs, 2])

                # metrics for inner_cv
                acc_inner, auc_inner, f1_inner = [], [], []
                test_acc_inner, test_auc_inner, test_f1_inner = [], [], []

                # inner cv is to tune the hyperparameters
                for train_index_inner, test_index_inner in inner_cv.split(inner_data, label_vec[train_index]):
                    x_train_inner, x_test_inner = inner_data[train_index_inner, :], inner_data[test_index_inner, :]
                    y_train_inner, y_test_inner = inner_label[train_index_inner, :], inner_label[test_index_inner, :]
                    with tf.Session(config=config) as sess:
                        sess.run(tf.global_variables_initializer())
                        total_batch = int(np.shape(x_train_inner)[0] / batch_size)

                        #earlystopping
                        best_cost = 1e9
                        best_acc = 0
                        best_f1 = 0
                        stop=False
                        last_improvement=0
                        required_improvement=50
                        costs=[]

                        acc, auc, f1 = [], [], []
                        # validate_acc, validate_auc = [], []
                        test_acc, test_auc, test_f1 = [], [], []

                        for epoch in range(training_epochs):
                            avg_cost = 0
                            # x_tmp, y_tmp = shuffle(x_train, y_train)
                            for k in range(total_batch - 1):
                                batch_x, batch_y = x_train_inner[k*batch_size:k*batch_size+batch_size], y_train_inner[k*batch_size:k*batch_size+batch_size]
                                _, c = sess.run([optimizer, cost], feed_dict={x:batch_x, y:batch_y, keep_prob:0.8, lr:learning_rate})
                                avg_cost += c/total_batch
                            # del x_tmp
                            # del y_tmp

                            #early stopping
                            # if avg_cost<best_cost:
                            # 	save_session = sess
                            # 	best_cost = avg_cost
                            # 	last_improvement=0
                            # else:
                            # 	last_improvement+=1
                            # if last_improvement>required_improvement:
                            # 	print("no improvement, early stopping applies")
                            # 	stop = True
                            # 	sess = save_session

                            acc_, y_s = sess.run([accuracy, y_score], feed_dict={x:x_train_inner, y:y_train_inner, keep_prob:1})
                            auc_ = metrics.roc_auc_score(y_train_inner, y_s)
                            y_train_label = np.argmax(y_train_inner, axis=1)
                            y_s_label = np.argmax(y_s, axis=1)
                            f1_ = metrics.f1_score(y_train_label, y_s_label)

                            acc.append(acc_)
                            auc.append(auc_)
                            f1.append(f1_)

                            # acc_inner.append(acc_)
                            # auc_inner.append(auc_)
                            # f1_inner.append(f1_)

                            # acc_, y_s = sess.run([accuracy, y_score], feed_dict={x: x_validate, y: y_validate, keep_prob: 1})
                            # auc_ = metrics.roc_auc_score(y_validate, y_s)

                            if acc_>best_acc:
                                save_session = sess
                                best_acc = acc_
                                last_improvement = 0
                            # if f1_ > best_f1:
                            #     save_session = sess
                            #     best_f1 = f1_
                            #     last_improvement = 0
                            else:
                                last_improvement +=1
                            if last_improvement>required_improvement:
                                print("stop at epoch", epoch)
                                print('final training acc, auc, f1:', acc_, auc_, f1_)
                                print("==============early stopping===================")
                                stop = True
                                sess = save_session

                            # validate_acc.append(acc_)
                            # validate_auc.append(auc_)

                            acc_, y_s = sess.run([accuracy, y_score], feed_dict={x:x_test_inner, y:y_test_inner, keep_prob:1})
                            auc_ = metrics.roc_auc_score(y_test_inner, y_s)
                            y_test_label = np.argmax(y_test_inner, axis=1)
                            y_s_label = np.argmax(y_s, axis=1)
                            f1_ = metrics.f1_score(y_test_label, y_s_label)

                            test_acc.append(acc_)
                            test_auc.append(auc_)
                            test_f1.append(f1_)

                            # test_acc_inner.append(acc_)
                            # test_auc_inner.append(auc_)
                            # test_f1_inner.append(f1_)

                            if stop is True:
                                break

                        print('final testing acc, auc, f1:', test_acc[-1], test_auc[-1], test_f1[-1])
                        print("==============next inner fold===================")
                        acc_inner.append(acc[-1])
                        auc_inner.append(auc[-1])
                        f1_inner.append(f1[-1])
                        test_acc_inner.append(test_acc[-1])
                        test_auc_inner.append(test_auc[-1])
                        test_f1_inner.append(test_f1[-1])

                avg_acc = np.array(acc_inner).mean()
                avg_auc = np.array(auc_inner).mean()
                avg_f1 = np.array(f1_inner).mean()
                avg_test_acc = np.array(test_acc_inner).mean()
                avg_test_auc = np.array(test_auc_inner).mean()
                avg_test_f1 = np.array(test_f1_inner).mean()

                std_acc = np.array(acc_inner).std()
                std_auc = np.array(auc_inner).std()
                std_f1 = np.array(f1_inner).std()
                std_test_acc = np.array(test_acc_inner).std()
                std_test_auc = np.array(test_auc_inner).std()
                std_test_f1 = np.array(test_f1_inner).std()

                arr = np.array([layers, layers_size, learning_rate, training_epochs, batch_size, avg_acc, std_acc, avg_test_acc, std_test_acc, avg_auc, std_auc, avg_test_auc, std_test_auc, avg_f1, std_f1, avg_test_f1, std_test_f1])
                log_inner.append(arr)

                print("<<<<<<<<<<<<<<<<<<<")
                print(str(layers_size))
                print("current inner cv process: "+str(count/all_exp_count))
                count+=1
                print("<<<<<<<<<<<<<<<<<<<")

    df_log_inner = pd.DataFrame(log_inner, columns = ['layers', 'layer_size', 'learning_rate', 'training_epochs', 'batch_size', 'train_acc', 'std', 'test_acc', 'std', 'train_auc', 'std', 'test_auc', 'std', 'train_f1', 'std', 'test_f1', 'std'])
    if fusion == 'none':
        filename2 = '_'
    elif fusion == 'net':
        filename2 = '_netfusion'
    elif fusion == 'feature':
        filename2 = '_feafusion'

    if feature_type == 'both':
        postfix = '_log.csv'
    elif feature_type == 'cen':
        postfix = '_cen_log.csv'
    elif feature_type == 'mod':
        postfix = '_mod_log.csv'
    else:
        print('output file naming error!')

    file2 = outpath + preflix + 'inner' + str(k_outer) + filename2 + postfix
    df_log_inner.to_csv(file2)

    best_idx = df_log_inner['test_f1'].idxmax()

    layers = df_log_inner['layers'].loc[best_idx]
    layers_size = df_log_inner['layer_size'].loc[best_idx]
    learning_rate = df_log_inner['learning_rate'].loc[best_idx]
    batch_size = df_log_inner['batch_size'].loc[best_idx]

    number_of_layers = 0
    for j in range(len(layers)):
        if layers[j] is True:
            number_of_layers = j+1

    n_hidden_1 = np.shape(expression)[1]
    n_classes = 2
    n_features = np.shape(expression)[1]

    x = tf.placeholder(tf.float32, [None, n_features])
    y = tf.placeholder(tf.float32, [None, n_classes])

    keep_prob = tf.placeholder(tf.float32)
    lr = tf.placeholder(tf.float32)

    weights = {
        'h1': tf.Variable(tf.truncated_normal(shape=[n_features, n_hidden_1], stddev=1/np.sqrt(n_features)))
    }

    biases = {
        'b1': tf.Variable(tf.zeros([n_hidden_1])),
        'out': tf.Variable(tf.zeros([n_classes]))
    }
    if layers[0]:
        weights['h2'] = tf.Variable(tf.truncated_normal(shape=[n_hidden_1, layers_size[0]], stddev=1/np.sqrt(n_hidden_1)))
        biases['b2'] = tf.Variable(tf.zeros([layers_size[0]]))
    if layers[1]:
        weights['h3'] = tf.Variable(tf.truncated_normal(shape=[layers_size[0], layers_size[1]], stddev=1/np.sqrt(layers_size[0])))
        biases['b3'] = tf.Variable(tf.zeros([layers_size[1]]))
    if layers[2]:
        weights['h4'] = tf.Variable(tf.truncated_normal(shape=[layers_size[1], layers_size[2]], stddev=1 / np.sqrt(layers_size[1])))
        biases['b4'] = tf.Variable(tf.zeros([layers_size[2]]))
    if layers[3]:
        weights['h5'] = tf.Variable(tf.truncated_normal(shape=[layers_size[2], layers_size[3]], stddev=1 / np.sqrt(layers_size[2])))
        biases['b5'] = tf.Variable(tf.zeros([layers_size[3]]))

    weights['out'] = tf.Variable(tf.truncated_normal(shape=[layers_size[number_of_layers-1], n_classes], stddev=1/np.sqrt(layers_size[number_of_layers-1])))

    pred_outer = dfn(x, layers, weights, biases, keep_prob)

    # cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=pred, labels=y))
    if dataset == 'SEQC':
        class_weights = tf.constant([1, 3.74])  #
    elif dataset == 'TARGET':
        class_weights = tf.constant([1, 1.04])  # 80/77
    else:
        print('wrong dataset type!')
        exit()
    cost = tf.reduce_mean(tf.nn.weighted_cross_entropy_with_logits(logits=pred_outer, targets=y, pos_weight=class_weights))
    if L2:
        reg = tf.nn.l2_loss(weights['h1'])
        if layers[0]:
            reg+=tf.nn.l2_loss(weights['h2'])
        if layers[1]:
            reg+=tf.nn.l2_loss(weights['h3'])
        if layers[2]:
            reg+=tf.nn.l2_loss(weights['h4'])
        if layers[3]:
            reg+=tf.nn.l2_loss(weights['h5'])
        reg+=tf.nn.l2_loss(weights['out'])
        cost = tf.reduce_mean(cost+0.001*reg)
    optimizer = tf.train.AdamOptimizer(learning_rate = lr).minimize(cost)

    correct_prediction = tf.equal(tf.argmax(pred_outer,1), tf.argmax(y,1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
    y_score = tf.nn.softmax(logits=pred_outer)

    loss_rec = np.zeros([training_epochs, 1])
    training_eval = np.zeros([training_epochs, 2])

    x_train, x_test = expression[train_index, :], expression[test_index, :]
    y_train, y_test = labels[train_index, :], labels[test_index, :]
    with tf.Session(config=config) as sess:
        sess.run(tf.global_variables_initializer())
        total_batch = int(np.shape(x_train)[0] / batch_size)

        #earlystopping
        best_cost = 1e9
        best_acc = 0
        stop=False
        last_improvement=0
        required_improvement=50
        costs=[]

        acc, auc, f1 = [], [], []
        # validate_acc, validate_auc = [], []
        test_acc, test_auc, test_f1 = [], [], []

        for epoch in range(training_epochs):
            avg_cost = 0
            # x_tmp, y_tmp = shuffle(x_train, y_train)
            for k in range(total_batch - 1):
                batch_x, batch_y = x_train[k*batch_size:k*batch_size+batch_size], y_train[k*batch_size:k*batch_size+batch_size]
                _, c = sess.run([optimizer, cost], feed_dict={x:batch_x, y:batch_y, keep_prob:0.8, lr:learning_rate})
                avg_cost += c/total_batch
            # del x_tmp
            # del y_tmp

            #early stopping
            # if avg_cost<best_cost:
            # 	save_session = sess
            # 	best_cost = avg_cost
            # 	last_improvement=0
            # else:
            # 	last_improvement+=1
            # if last_improvement>required_improvement:
            # 	print("no improvement, early stopping applies")
            # 	stop = True
            # 	sess = save_session

            acc_, y_s = sess.run([accuracy, y_score], feed_dict={x:x_train, y:y_train, keep_prob:1})
            auc_ = metrics.roc_auc_score(y_train, y_s)
            y_train_label = np.argmax(y_train, axis=1)
            y_s_label = np.argmax(y_s, axis=1)
            f1_ = metrics.f1_score(y_train_label, y_s_label)

            acc.append(acc_)
            auc.append(auc_)
            f1.append(f1_)

            # acc_cv.append(acc_)
            # auc_cv.append(auc_)
            # f1_cv.append(f1_)

            # acc_, y_s = sess.run([accuracy, y_score], feed_dict={x: x_validate, y: y_validate, keep_prob: 1})
            # auc_ = metrics.roc_auc_score(y_validate, y_s)

            if acc_>best_acc:
                save_session = sess
                best_acc = acc_
                last_improvement = 0
            else:
                last_improvement +=1
            if last_improvement>required_improvement:
                print("==============early stopping===================")
                stop = True
                sess = save_session

            # validate_acc.append(acc_)
            # validate_auc.append(auc_)

            acc_, y_s = sess.run([accuracy, y_score], feed_dict={x:x_test, y:y_test, keep_prob:1})
            auc_ = metrics.roc_auc_score(y_test, y_s)
            y_test_label = np.argmax(y_test, axis=1)
            y_s_label = np.argmax(y_s, axis=1)
            f1_ = metrics.f1_score(y_test_label, y_s_label)

            test_acc.append(acc_)
            test_auc.append(auc_)
            test_f1.append(f1_)

            # test_acc_outer.append(acc_)
            # test_auc_outer.append(auc_)
            # test_f1_outer.append(f1_)

            if stop is True:
                break

        acc_outer.append(acc[-1])
        auc_outer.append(auc[-1])
        f1_outer.append(f1[-1])
        test_acc_outer.append(test_acc[-1])
        test_auc_outer.append(test_auc[-1])
        test_f1_outer.append(test_f1[-1])

    arr = np.array([k_outer, layers_size, learning_rate, training_epochs, batch_size, acc_outer[-1], auc_outer[-1], f1_outer[-1]])
    log_outer.append(arr)
    print('<<<<<<<<<<<<<<<<< FINISH OUTER FOLD', k_outer, '<<<<<<<<<<<<<<<<<<<')

avg_acc = np.array(acc_outer).mean()
avg_auc = np.array(auc_outer).mean()
avg_f1 = np.array(f1_outer).mean()
avg_test_acc = np.array(test_acc_outer).mean()
avg_test_auc = np.array(test_auc_outer).mean()
avg_test_f1 = np.array(test_f1_outer).mean()

std_acc = np.array(acc_outer).std()
std_auc = np.array(auc_outer).std()
std_f1 = np.array(f1_outer).std()
std_test_acc = np.array(test_acc_outer).std()
std_test_auc = np.array(test_auc_outer).std()
std_test_f1 = np.array(test_f1_outer).std()

print('final results:', avg_acc, std_acc, avg_test_acc, std_test_acc, avg_auc, std_auc, avg_test_auc, std_test_auc, avg_f1, std_f1, avg_test_f1, std_test_f1)
df_log_outer = pd.DataFrame(log_outer, columns = ['k_outer', 'layer_size', 'lr', 'training_epochs', 'bs', 'acc', 'auc', 'f1'])
file3 = outpath + preflix + 'outer' + filename2 + postfix

df_log_outer.to_csv(file3)


import pandas as pd

class lift_boolean:
      """take the difference between two time columns"""
      def __init__(self,actual_df,lift_df,lift_threshold):
         self.actual_df = actual_df # array of column names to encode
         self.lift_df = lift_df
         self.lift_threshold = lift_threshold
         """
         This function creates a boolean feature   for a given categorical feature
         by selecting those category levels that provide the highest lift ratio.
         The lift ratio is calculated by dividing the percent of fraud transactions by the
         percent of clean transactions.
         It appends the new boolean feature to the old dataframe that was given as the input
         to the function.
         """
      def create_boolean(self):

          self.lift_df = self.lift_df[self.lift_df.Lift_Ratio > self.lift_threshold ]
          cols =   self.lift_df.feature_name.unique()
          cols_2 = list(set(self.actual_df.columns) & set(cols))
          for i in cols_2:
            #create a boolean column in Data  if the  column i  elements are in the lift category list
              self.actual_df['Bool_'+ i]= [1 if self.actual_df[i].values.tolist()[j] in self.lift_df.Category_levels.to_list() else 0 for j in range(self.actual_df.shape[0])]
             #Counter(Data['Bool_'+ i])
          return self.actual_df


import pya
import os
from .draw_mos import *
from .mos import mos_ld, mos_grw

USER = os.environ['USER']
gds_path = f"/home/{USER}/.klayout/pymacros/cells/efuse"

def draw_efuse(layout):

    layout.read(f"{gds_path}/efuse.gds")
    #layout.read(f"{gds_path}/efuse_bitline.gds")
    cell_name = "efuse_cell"

    return layout.cell(cell_name)

def draw_efuse_senseamp(layout):

    layout.read(f"{gds_path}/efuse_senseamp.gds")
    cell_name = "efuse_senseamp_cell"
    layout.cell(cell_name).shapes(layout.layer(63, 63)).clear()
    metal1_label = layout.layer(34, 10)

    layout.cell(cell_name).shapes(metal1_label).clear()

    return layout.cell(cell_name)

def draw_efuse_bitline(layout, n_bitcells=4):
    n_full_lines = n_bitcells // 4
    n_extra_cells = n_bitcells % 4
    efuse_bitline_index = layout.add_cell("efuse_bitline_cell")
    efuse_bitline_cell = layout.cell(efuse_bitline_index)
    metal1_label = layout.layer(34, 10)
    metal2 = layout.layer(36, 0)
    metal2_label = layout.layer(36, 10)
    metal4_label = layout.layer(46, 10)
    line_sel_pin = pya.Point(1820, 51385)

    cur_pos = 0
    if n_full_lines > 0:
        layout.read(f"{gds_path}/efuse_bitline4.gds")
        efuse_bitline_instance = layout.cell("efuse_bitline4_cell")
        efuse_bitline_bbox = efuse_bitline_instance.bbox()
        efuse_bitline_width = efuse_bitline_bbox.right - efuse_bitline_bbox.left
        efuse_bitline_spacing = -920
        efuse_bitline_step = efuse_bitline_width + efuse_bitline_spacing
        full_bitlines = pya.CellInstArray(efuse_bitline_instance.cell_index(), pya.Trans(pya.Point(0, 0)),
                                        pya.Vector(efuse_bitline_step, 0), pya.Vector(0, 0),
                                        n_full_lines, 1)
        efuse_bitline_cell.insert(full_bitlines)

        line_sel_wire = pya.Box(line_sel_pin-pya.Vector(210, 210), line_sel_pin+pya.Vector(efuse_bitline_step*(n_full_lines-1), 210))
        efuse_bitline_cell.shapes(metal2).insert(line_sel_wire)
        cur_pos = efuse_bitline_step*(n_full_lines)

    if n_extra_cells > 0:
        layout.read(f"{gds_path}/efuse_bitline{n_extra_cells}.gds")
        extra_cells_instance = layout.cell(f"efuse_bitline{n_extra_cells}_cell")
        extra_cells = pya.CellInstArray(extra_cells_instance.cell_index(), pya.Trans(pya.Point(cur_pos, 0)))
        efuse_bitline_cell.insert(extra_cells)

    if n_full_lines > 0 and n_extra_cells > 0:
        line_sel_wire = pya.Box(line_sel_pin-pya.Vector(210, 210), line_sel_pin+pya.Vector(cur_pos, 210))
        efuse_bitline_cell.shapes(metal2).insert(line_sel_wire)

    efuse_bitline_cell.flatten(1)
    efuse_bitline_cell.shapes(metal1_label).clear()
    efuse_bitline_cell.shapes(metal2_label).clear()
    efuse_bitline_cell.shapes(metal4_label).clear()

    return efuse_bitline_cell

def draw_efuse_array(layout, n_bitlines=4, n_bitcells=4):

    # define layers
    contact = layout.layer(33, 0)
    metal1 = layout.layer(34, 0)
    metal1_label = layout.layer(34, 10)
    via1 = layout.layer(35, 0)
    metal2 = layout.layer(36, 0)
    metal2_label = layout.layer(36, 10)
    via2 = layout.layer(38, 0)
    metal3 = layout.layer(42, 0)
    metal3_label = layout.layer(42, 10)
    via3 = layout.layer(40, 0)
    metal4 = layout.layer(46, 0)
    metal4_label = layout.layer(46, 10)
    via4 = layout.layer(41, 0)
    metal5 = layout.layer(81, 0)
    poly2 = layout.layer(30, 0)
    dualgate = layout.layer(55, 0)

    # global parameters
    wire_width = 330
    anode_wire_width = 1650
    anode_wire_sep = 1000
    metalvia_overlap = 60
    via_size = 260
    rail_width = 2500
    vss_rail_y = 51500
    vdd_rail_y = -10000

    def place_via_tower(cell: pya.Cell, point: pya.Point, bottom_metal: int, top_metal: int):
        """
        Place necessary vias metals connecting a point on bottom_metal to the same point on top_metal.
        """
        metals = {1: metal1, 2: metal2, 3: metal3, 4: metal4, 5: metal5}
        vias = {1: via1, 2: via2, 3: via3, 4: via4}
        via_box = pya.Box(point.x-via_size/2, point.y-via_size/2, point.x+via_size/2, point.y+via_size/2)
        metal_box = pya.Box(point.x-via_size/2-metalvia_overlap, point.y-via_size/2-metalvia_overlap, point.x+via_size/2+metalvia_overlap, point.y+via_size/2+metalvia_overlap)
        cell.shapes(metals[bottom_metal]).insert(metal_box)
        for i in range(bottom_metal, top_metal):
            cell.shapes(vias[i]).insert(via_box)
            cell.shapes(metals[i+1]).insert(metal_box)

    # create cell
    efuse_array_index = layout.add_cell("efuse_array")
    efuse_array_cell = layout.cell(efuse_array_index)

    bitline_instance = draw_efuse_bitline(layout, n_bitcells=n_bitcells)
    bitline_bbox = bitline_instance.bbox()
    bitline_width = bitline_bbox.right - bitline_bbox.left
    bitline_spacing = 500 #-160-320# they need to overlap a bit
    bitline_step = bitline_width + bitline_spacing
    full_line_width = 21495
    anode0_pin = pya.Point( 5600, 39800)
    anode1_pin = pya.Point( 5600, 11800)
    anode2_pin = pya.Point(15900, 11800)
    anode3_pin = pya.Point(15900, 39800)
    anode_pins = [pin + pya.Vector((full_line_width-920)*i, 0) for i in range(n_bitcells//4 + 1)
                                for pin in [anode0_pin, anode1_pin, anode2_pin, anode3_pin]][:n_bitcells]
    line_sel_pin = pya.Point(1820, 51385) # take the leftmost point
    vss_0_pin = pya.Point(455, 50200)
    vss_1_pin = pya.Point(10745, 50200)
    vss_2_pin = pya.Point(21030, 50200)
    n_bitline_vss_pins = (n_bitcells//4)*3 + {0: 0, 1: 1, 2: 2, 3: 2}[n_bitcells%4]
    vss_pins = [pin + pya.Vector((full_line_width-920)*i, 0) for i in range(n_bitcells//4 + 1)
                                for pin in [vss_0_pin, vss_1_pin, vss_2_pin]][:n_bitline_vss_pins]

    bitlines = pya.CellInstArray(bitline_instance.cell_index(), pya.Trans(pya.Point(0, 0)),
                                    pya.Vector(bitline_step, 0), pya.Vector(0, 0),
                                    n_bitlines, 1)
    efuse_array_cell.insert(bitlines)

    PMOS = draw_pmos(layout, 0.5, 50, mos_ld, 1, mos_grw, "Bulk Tie", "5V", 0, 0)
    PMOS_width = 2660
    PMOS_spacing = 1980
    PMOS_step = PMOS_width+PMOS_spacing
    PMOS_bulk_pin_x = -580
    PMOS_source_pin_x = 180
    PMOS_drain_pin_x = 1205
    PMOS_bulk_drain_pin_y = 500
    PMOS_gate_pin = pya.Point(690, PMOS.bbox().height()-220)
    PMOS_gate_extension = pya.Box(440, 50220, 940, 50620)
    PMOS_dualgate_extension = pya.Box(-1040, 50620, 1620, 51120)
    bitline_PMOS_sep = 960 + 500
    PMOS_start_x = bitline_step*(n_bitlines)+bitline_PMOS_sep
    PMOS_y = 620
    PMOSs = pya.CellInstArray(PMOS.cell_index(), pya.Trans(pya.Point(PMOS_start_x, PMOS_y)),
                                pya.Vector(PMOS_step, 0), pya.Vector(0, 0),
                                n_bitcells, 1)
    efuse_array_cell.insert(PMOSs)

    senseamp_instance = draw_efuse_senseamp(layout)
    senseamp_bbox = senseamp_instance.bbox()
    senseamp_width = senseamp_bbox.right - senseamp_bbox.left
    senseamp_height = senseamp_bbox.top - senseamp_bbox.bottom
    senseamp_spacing = 1350
    senseamp_step = senseamp_width + senseamp_spacing
    senseamp_vss_pin = pya.Point(6000, -400)
    senseamp_vdd_pin = pya.Point(5400, -4350)
    senseamp_out_pin = pya.Point(9400, -2390)
    fuse_pin = pya.Point(1920, -1020)
    npreset_pin = pya.Point(1890, -2360)
    sense_pin = pya.Point(1480, -1700)

    senseamp_y = vdd_rail_y + 7000
    senseamps = pya.CellInstArray(senseamp_instance.cell_index(), pya.Trans(pya.Point(0, senseamp_y)),
                                    pya.Vector(senseamp_step, 0), pya.Vector(0, 0),
                                    n_bitcells, 1)
    efuse_array_cell.insert(senseamps)

    leftmost_sense_pin = sense_pin + pya.Vector(0, senseamp_y)
    rightmost_sense_pin = sense_pin + pya.Vector(senseamp_step*(n_bitcells-1), senseamp_y)
    sense_wire = pya.Box(leftmost_sense_pin - pya.Vector(wire_width/2, wire_width/2), rightmost_sense_pin + pya.Vector(wire_width/2, wire_width/2))
    leftmost_npreset_pin = npreset_pin + pya.Vector(0, senseamp_y)
    rightmost_npreset_pin = npreset_pin + pya.Vector(senseamp_step*(n_bitcells-1), senseamp_y)
    npreset_wire = pya.Box(leftmost_npreset_pin - pya.Vector(wire_width/2, wire_width/2), rightmost_npreset_pin + pya.Vector(wire_width/2, wire_width/2))
    efuse_array_cell.shapes(metal3).insert(sense_wire)
    efuse_array_cell.shapes(metal3).insert(npreset_wire)
    efuse_array_cell.shapes(metal3_label).insert(pya.Text("SENSE", pya.Trans(sense_wire.center())))
    efuse_array_cell.shapes(metal3_label).insert(pya.Text("nPRESET", pya.Trans(npreset_wire.center())))

    stripe_right = efuse_array_cell.bbox().right
    stripe_left  = efuse_array_cell.bbox().left

    # add metal4 vss rail
    vss_rail = pya.Box(stripe_left, vss_rail_y-rail_width/2, stripe_right, vss_rail_y+rail_width/2)
    efuse_array_cell.shapes(metal4).insert(vss_rail)
    efuse_array_cell.shapes(metal4_label).insert(pya.Text("VSS", pya.Trans(vss_rail.center())))

    # add metal4 vdd rail
    vdd_rail = pya.Box(stripe_left, vdd_rail_y-rail_width/2, stripe_right, vdd_rail_y+rail_width/2)
    efuse_array_cell.shapes(metal4).insert(vdd_rail)
    efuse_array_cell.shapes(metal4_label).insert(pya.Text("VDD", pya.Trans(vdd_rail.center())))

    # create metal stripes connecting corresponding anodes
    anode_wires_y = []
    anode_wires = []
    top_wires = 0
    bottom_wires = 0
    for i in range(n_bitcells):
        if i%4 in [0, 3]:
            anode_wires_y.append(38600 - (anode_wire_width + anode_wire_sep)*top_wires)
            top_wires += 1
        else:
            anode_wires_y.append(13000 + (anode_wire_width + anode_wire_sep)*bottom_wires)
            bottom_wires += 1
        anode_wires.append(pya.Box(0, anode_wires_y[i]-anode_wire_width/2, stripe_right, anode_wires_y[i]+anode_wire_width/2))
        efuse_array_cell.shapes(metal3).insert(anode_wires[i])

    for i in range(n_bitcells):
        # connect PMOS drain and bulk to VDD
        current_PMOS_displ = pya.Vector(PMOS_start_x+PMOS_step*i, 0)
        current_bulk_pin = pya.Point(PMOS_bulk_pin_x, PMOS_bulk_drain_pin_y) + current_PMOS_displ
        current_drain_pin = pya.Point(PMOS_drain_pin_x, PMOS_bulk_drain_pin_y) + current_PMOS_displ
        vdd_wire = pya.Box(current_bulk_pin.x-wire_width/2, vdd_rail.bottom, current_drain_pin.x+wire_width/2, vss_rail.bottom-2000)
        efuse_array_cell.shapes(metal4).insert(vdd_wire)
        via_step = metalvia_overlap*2+via_size+280
        for k in range((min(anode_wires_y)-current_bulk_pin.y-anode_wire_width-500)//via_step):
            place_via_tower(efuse_array_cell, pya.Point(current_bulk_pin.x, current_bulk_pin.y+via_step*k), 1, 4)
            place_via_tower(efuse_array_cell, pya.Point(current_drain_pin.x, current_drain_pin.y+via_step*k), 1, 4)
        for k in range((vss_rail.bottom-2500-max(anode_wires_y)-anode_wire_width-500)//via_step):
            place_via_tower(efuse_array_cell, pya.Point(current_bulk_pin.x, vss_rail.bottom-2500-via_step*k), 1, 4)
            place_via_tower(efuse_array_cell, pya.Point(current_drain_pin.x, vss_rail.bottom-2500-via_step*k), 1, 4)
        # extend gate and label PMOS gate pin
        efuse_array_cell.shapes(poly2).insert(PMOS_gate_extension.moved(pya.Vector(bitline_step*(n_bitlines)+bitline_PMOS_sep+PMOS_step*i, 620)))
        efuse_array_cell.shapes(dualgate).insert(PMOS_dualgate_extension.moved(pya.Vector(bitline_step*(n_bitlines)+bitline_PMOS_sep+PMOS_step*i, 620)))
        current_gate_pin = PMOS_gate_pin + current_PMOS_displ
        efuse_array_cell.shapes(contact).insert(pya.Box(current_gate_pin-pya.Vector(110, 110), current_gate_pin+pya.Vector(110,110)))
        efuse_array_cell.shapes(metal1).insert(pya.Box(current_gate_pin-pya.Vector(170, 170), current_gate_pin+pya.Vector(170,330)))
        efuse_array_cell.shapes(metal1_label).insert(pya.Text(f"COL_PROG[{i}]", pya.Trans(current_gate_pin)))
        # connect source pin to anode_wire
        place_via_tower(efuse_array_cell, pya.Point(PMOS_source_pin_x, anode_wires_y[i]) + current_PMOS_displ, 1, 3)

        # connect senseamp pins
        current_senseamp_displ = pya.Vector(senseamp_step*i, senseamp_y)
        current_vss_pin = senseamp_vss_pin + current_senseamp_displ
        vss_wire = pya.Box(current_vss_pin.x-wire_width/2, current_vss_pin.y-wire_width/2, current_vss_pin.x+wire_width/2, vss_rail.top)
        efuse_array_cell.shapes(metal4).insert(vss_wire)
        place_via_tower(efuse_array_cell, current_vss_pin, 1, 4)

        current_vdd_pin = senseamp_vdd_pin + current_senseamp_displ
        vdd_wire = pya.Box(current_vdd_pin.x-wire_width/2, vdd_rail.bottom, current_vdd_pin.x+wire_width/2, current_vdd_pin.y+wire_width/2)
        efuse_array_cell.shapes(metal4).insert(vdd_wire)
        place_via_tower(efuse_array_cell, current_vdd_pin, 1, 4)

        current_out_pin = senseamp_out_pin + current_senseamp_displ
        efuse_array_cell.shapes(metal1_label).insert(pya.Text(f"DO[{i}]", pya.Trans(current_out_pin)))

        current_fuse_pin = fuse_pin + current_senseamp_displ
        fuse_wire = pya.Box(current_fuse_pin.x-wire_width/2, anode_wires_y[i]-wire_width/2, current_fuse_pin.x+wire_width/2, current_fuse_pin.y+wire_width/2)
        efuse_array_cell.shapes(metal4).insert(fuse_wire)
        place_via_tower(efuse_array_cell, current_fuse_pin, 1, 4)
        place_via_tower(efuse_array_cell, pya.Point(current_fuse_pin.x, anode_wires_y[i]), 3, 4)

        current_sense_pin = sense_pin + current_senseamp_displ
        place_via_tower(efuse_array_cell, current_sense_pin, 1, 3)

        current_npreset_pin = npreset_pin + current_senseamp_displ
        place_via_tower(efuse_array_cell, current_npreset_pin, 1, 3)

    for i in range(n_bitlines):
        # connect anodes to routing metal3 (anode_wires)
        current_bitline_displ = pya.Vector(bitline_step*i, 0)
        for j in range(n_bitcells):
            current_anode_pin = anode_pins[j] + current_bitline_displ
            if anode_pins[j].y > anode_wires_y[j]:
                anode_extension = pya.Box(current_anode_pin.x-anode_wire_width/2, anode_wires_y[j]-anode_wire_width/2,
                                            current_anode_pin.x+anode_wire_width/2, anode_pins[j].y+anode_wire_width/2)
                efuse_array_cell.shapes(metal1).insert(anode_extension)
            else:
                anode_extension = pya.Box(current_anode_pin.x-anode_wire_width/2, anode_pins[j].y-anode_wire_width/2,
                                            current_anode_pin.x+anode_wire_width/2, anode_wires_y[j]+anode_wire_width/2)
                efuse_array_cell.shapes(metal1).insert(anode_extension)
            place_via_tower(efuse_array_cell, pya.Point(current_anode_pin.x, anode_wires_y[j]), 1, 3)

        # place line_i pin
        current_line_sel_pin = line_sel_pin + pya.Vector(bitline_step*i, 0)
        efuse_array_cell.shapes(metal2_label).insert(pya.Text(f"LINE[{i}]", pya.Trans(current_line_sel_pin)))

        # connect to VSS
        for j in range(n_bitline_vss_pins):
            current_vss_pin = vss_pins[j] + current_bitline_displ
            # vss_connect = pya.Box(current_vss_pin.x-rail_width/2, 0, current_vss_pin.x+rail_width/2, vss_rail.top)
            vss_connect = pya.Box(current_vss_pin.x-rail_width/2, max(anode_wires_y)+anode_wire_width/2,
                                    current_vss_pin.x+rail_width/2, vss_rail.top)
            efuse_array_cell.shapes(metal4).insert(vss_connect)
            via_step = metalvia_overlap*2+via_size+280
            for k in range((current_vss_pin.y-max(anode_wires_y)-anode_wire_width-500)//via_step):
                place_via_tower(efuse_array_cell, current_vss_pin+pya.Vector(0, -k*via_step), 1, 4)
            # for k in range((min(anode_wires_y)-1130-anode_wire_width-500)//via_step):
            #     place_via_tower(efuse_array_cell, pya.Point(current_vss_pin.x, 1130+k*via_step), 1, 4)

    return efuse_array_cell