#!/usr/bin/python
#File: lexical_simplification.py
#By: Boyd Belshof, S3012158
#Date: 17-12-19

import pickle, pyphen, operator, openpyxl
import pandas as pd
from operator import itemgetter
from stop_words import get_stop_words
from svm_wsd.dsc_wsd_tagger import parse_sentence
from cornetto.cornet import Cornet
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import RandomForestRegressor
from sklearn import metrics

# Load Cornetto Library
c = Cornet()
c.open("cornetto/cdb2.0.lu.stripped.xml", "cornetto/cdb2.0.syn.stripped.xml")

#Load Stopwords File
stop_words = get_stop_words('nl')

#Loads Pyphen Library
dic = pyphen.Pyphen(lang='nl_NL')

# Loads Dictionaries
with open('aoa.pickle', 'rb') as handle:
		aoa_dict = pickle.load(handle)

with open('wordcounter.pickle', 'rb') as handle:
		word_dict = pickle.load(handle)

with open('transformation.pickle', 'rb') as handle:
		transformation_dict = pickle.load(handle)

with open('troonrede_training_data.pickle', 'rb') as handle:
		training_dict = pickle.load(handle)

with open('../wordcounter_plain_python2.pickle', 'rb') as handle:
		word_freq_dict = pickle.load(handle)

data = pd.DataFrame(training_dict,columns=['aoa', 'count', 'len_', 'syl', 'syn', 'near_syn', 'hyper', 'hypo', 'label'])
X = data[['aoa', 'count', 'len_', 'syl', 'syn', 'near_syn', 'hyper', 'hypo']]  # Features
y = data['label']
clf = RandomForestClassifier(n_estimators=100)
clf.fit(X,y)

def is_stopword(word):
	if word in stop_words:
		return True
	else:
		return False

def create_pos_tags(tag):
	if tag in ['verbinf', 'verbpapa', 'verbpastpl', 'verbpastsg', 'verbpresp', 'verprespl', 'verbpressg', 'verb']:
		return 'verb'

	elif tag in ['noun*kop', 'nounabbr', 'nounpl', 'nounprop', 'noungsg', 'nounsg', 'prondemo', 'pronindef', 'pronposs', 'pronquest', 'pronrefl', 'pronrel']:
		return 'noun'

	elif tag in ['adj', 'adj*kop', 'adjabbr']:
		return 'adj'

	else:
		return None;

def get_aoa(word):
	if word in aoa_dict:
		return aoa_dict[word]
	else:
		return 0

def word_by_freq(word):
	return_synonym = word
	for synonym in synonyms:
		if return_synonym in word_dict and synonym in word_dict:
			if word_dict[return_synonym] < word_dict[synonym]:
				return_synonym = synonym
		elif return_synonym not in word_dict and synonym in word_dict:
			return_synonym = synonym

	return return_synonym

def get_related_words(x, method):
	# 0 = BY ID
	# 1 = BY TOKEN
	if method == 0:
		sense = c.get_lex_unit_by_id(x)
	else:
		sense = x
	related_items = c.get_related_lex_units(sense, '1')
	list_of_items = []
	if sense in related_items:
		if 'SYNONYM' in related_items[sense]:
			for item in related_items[sense]['SYNONYM']: # HIER KUNNEN ER NOG MEER ACHTER
				list_of_items.append(item)
		if 'NEAR_SYNONYM' in related_items[sense]:
			for item in related_items[sense]['NEAR_SYNONYM']: # HIER KUNNEN ER NOG MEER ACHTER
				list_of_items.append(item)
		if 'HAS_HYPONYM' in related_items[sense]:
			for item in related_items[sense]['HAS_HYPONYM']: # HIER KUNNEN ER NOG MEER ACHTER
				list_of_items.append(item)
	return list_of_items

def get_right_form(lemma, tree_tagger_pos_tag):
	if lemma in transformation_dict:
		if tree_tagger_pos_tag in transformation_dict[lemma]:
			result = transformation_dict[lemma][tree_tagger_pos_tag]
			key = max(result.iteritems(), key=operator.itemgetter(1))[0]
			return key
	return lemma

def generate_key(lemma, pos_tag):
	if create_pos_tags(pos_tag) is not None:
		key = lemma + ":" + create_pos_tags(pos_tag) + ":1"
	else:
		key = lemma + ":1"
	return key

def keywithmaxval(d):
	highest_key = None
	for key in d:
		if highest_key is None:
			highest_key = key

		old = d[key][0][0]
		new = d[highest_key][0][0]
		if old > new:
			highest_key = key

	return highest_key, d[highest_key]

def should_simplify(word, lemma, pos_tag, key, _id, final_results):
	# PLAIN TEXT TODO
	list_of_synonyms = []

	if lemma in aoa_dict:
		aoa = (aoa_dict[lemma])

	count, list_of_synonyms = get_count_and_synonyms(_id, final_results, pos_tag, lemma, key)
	o_aoa = o_count = o_len_ = o_syl = o_syn = o_near_syn = o_hyper = o_hypo = 0
	if lemma in word_freq_dict:
		o_count = word_freq_dict[lemma]
	if lemma in aoa_dict:
		o_aoa = aoa_dict[lemma]
	o_syl = len(dic.inserted(lemma).split('-'))
	o_len_ = len(lemma)

	if _id in final_results:
		syn, near_syn, hyper, hypo = get_wordnet_features(final_results[_id][0], 0)

	# If not given create a WSD key
	else:
		new_tag = create_pos_tags(pos_tag)
		if new_tag is not None:
			key_list = c.get_lex_units(lemma + ":" + new_tag)
		else:
			key_list = c.get_lex_units(lemma)
			if len(key_list) > 0:
				key = key_list[0]
				o_syn, o_near_syn, o_hyper, o_hypo = get_wordnet_features(key, 1)

	prediction = clf.predict([[o_aoa, o_count, o_len_, o_syl, o_syn, o_near_syn, o_hyper, o_hypo]])

	if prediction[0] == 1:
		return True
	else:
		return False

def get_wordnet_features(x, method):
	# 0 = BY ID
	# 1 = BY TOKEN
	if method == 0:
		sense = c.get_lex_unit_by_id(x)
	else:
		sense = x
	related_items = c.get_related_lex_units(sense, '1')
	syn = near_syn = hypo = hyper = 0
	if sense in related_items:
		if 'SYNONYM' in related_items[sense]:
			for item in related_items[sense]['SYNONYM']: # HIER KUNNEN ER NOG MEER ACHTER
				syn += 1
		if 'NEAR_SYNONYM' in related_items[sense]:
			for item in related_items[sense]['NEAR_SYNONYM']: # HIER KUNNEN ER NOG MEER ACHTER
				near_syn += 1
		if 'HAS_HYPERONYM' in related_items[sense]:
			for item in related_items[sense]['HAS_HYPERONYM']: # HIER KUNNEN ER NOG MEER ACHTER
				hyper += 1
		if 'HAS_HYPONYM' in related_items[sense]:
			for item in related_items[sense]['HAS_HYPONYM']: # HIER KUNNEN ER NOG MEER ACHTER
				hypo += 1
	return syn, near_syn, hyper, hypo

def get_count_and_synonyms(_id, final_results, pos_tag, lemma, key):
	count = 0
	list_of_synonyms = []
	# Check if key is given by WSD
	if _id in final_results:
		list_of_synonyms = get_related_words(final_results[_id][0], 0)
		wordnet_features = len(list_of_synonyms)
		if key in word_dict:
			count = word_dict[key]

	# If not given create a WSD key
	else:
		new_tag = create_pos_tags(pos_tag)
		if new_tag is not None:
			key_list = c.get_lex_units(lemma + ":" + new_tag)
		else:
			key_list = c.get_lex_units(lemma)
			if len(key_list) > 0:
				key = key_list[0]
				list_of_synonyms = get_related_words(key, 1)
				if key in word_dict:
					count = word_dict[key]

	return count, list_of_synonyms

def get_simplification(word, lemma, pos_tag, key, _id, final_results):
	# PLAIN TEXT TODO
	list_of_synonyms = []

	if lemma in aoa_dict:
		aoa = (aoa_dict[lemma])

	count, list_of_synonyms = get_count_and_synonyms(_id, final_results, pos_tag, lemma, key)
	o_aoa = o_count = o_len_ = o_syl = o_syn = o_near_syn = o_hyper = o_hypo = 0
	if lemma in word_freq_dict:
		o_count = word_freq_dict[lemma]
	if lemma in aoa_dict:
		o_aoa = aoa_dict[lemma]
	o_syl = len(dic.inserted(lemma).split('-'))
	o_len_ = len(lemma)

	if _id in final_results:
		syn, near_syn, hyper, hypo = get_wordnet_features(final_results[_id][0], 0)

	# If not given create a WSD key
	else:
		new_tag = create_pos_tags(pos_tag)
		if new_tag is not None:
			key_list = c.get_lex_units(lemma + ":" + new_tag)
		else:
			key_list = c.get_lex_units(lemma)
			if len(key_list) > 0:
				key = key_list[0]
				o_syn, o_near_syn, o_hyper, o_hypo = get_wordnet_features(key, 1)

	original = clf.predict_proba([[o_aoa, o_count, o_len_, o_syl, o_syn, o_near_syn, o_hyper, o_hypo]])

	if len(list_of_synonyms) > 0:
		synonyms = {}
		for synonym in list_of_synonyms:
			aoa = count = len_ = syl = syn = near_syn = hyper = hypo = 0
			synonym_lemma = synonym.split(':')[0]

			if synonym_lemma in word_freq_dict:
				count = word_freq_dict[synonym_lemma]
			if synonym_lemma in aoa_dict:
				aoa = aoa_dict[synonym_lemma]

			syllables_num = len(dic.inserted(synonym_lemma).split('-'))
			word_len = len(synonym_lemma)

			len_ = len(synonym_lemma)
			syl = len(dic.inserted(word).split('-'))

			new_tag = create_pos_tags(pos_tag)
			if new_tag is not None:
				key_list = c.get_lex_units(synonym_lemma + ":" + new_tag)
			else:
				key_list = c.get_lex_units(synonym_lemma)
				if len(key_list) > 0:
					key = key_list[0]
					syn, near_syn, hyper, hypo = get_wordnet_features(key, 1)

			synonyms[synonym_lemma] = clf.predict_proba([[aoa, count, len_, syl, syn, near_syn, hyper, hypo]])

		syn_key, syn_val = keywithmaxval(synonyms)
		value = syn_val[0][0]

		if (original[0][0] < value):
			return syn_key, True
		else:
			return word, False
	return word, False

def simplify_sentence(sentence):
	tokens, final_results = parse_sentence(sentence)
	return_sentence = []
	for token in tokens:
		_id, word, pos_tag, lemma, number = token
		aoa = count = 0
		# Generate Key
		key = generate_key(lemma, pos_tag)
		old_word = word

		# Check for stopwords and pronouns
		if should_simplify(word, lemma, pos_tag, key, _id, final_results):
			word, new = get_simplification(word, lemma, pos_tag, key, _id, final_results)
			if word is not None:
				if new is True:
					return_sentence.append(get_right_form(word, pos_tag))
				if new is False:
					return_sentence.append(old_word)
			else:
				return_sentence.append(old_word)
		else:
			return_sentence.append(word)
	actual_sentence = ""
	for word in return_sentence:
		actual_sentence += word + ' '

	return actual_sentence

def main():
	while True:
		input_sentence = raw_input()
		print(simplify_sentence(input_sentence))

if __name__ == '__main__':
	main()


# Service for reading and writing to IO.
class fileIOService:

    #Reads from a file and returns file object.
    def read(self, filename):
         fileObject = open(filename, 'r')
         data = fileObject.read()
         fileObject.close()

         return data

    #Returns a file object for writing.
    def write(self, filename, data):
        fileObject = open(filename, 'w+')
        fileObject.write(data)
        fileObject.close()


import discogs_client, random
d = discogs_client.Client('kesmit/0.1', user_token='---YOUR_API_TOKEN_GOES_HERE---')
me = d.identity()
albums = []
print('Fetching collection information from Discogs...')
print('Skipping singles, 7", 10"...')
for item in me.collection_folders[0].releases:
	try:
		if not "7\"" in item.release.formats[0]['descriptions'] and not "10\"" in item.release.formats[0]['descriptions']:
			albums.append(item)
	except KeyError:
		continue
		# print(f"skipping {item}...")

print(f"{len(albums)} albums in collection")

random_number = random.randint(0,len(albums)-1)

album_artist = albums[random_number].release.artists[0].name
album_title = albums[random_number].release.title

print(f'\nYour random selection is :\n\t >>> {album_artist} - {album_title} <<<\n')


from micropython import const
from typing import TYPE_CHECKING

from trezor.wire import DataError

from .. import writers

if TYPE_CHECKING:
    from trezor.messages import TxAckPaymentRequest, TxOutput

    from apps.common import coininfo
    from apps.common.keychain import Keychain

_MEMO_TYPE_TEXT = const(1)
_MEMO_TYPE_REFUND = const(2)
_MEMO_TYPE_COIN_PURCHASE = const(3)

class PaymentRequestVerifier:
    if __debug__:
        # secp256k1 public key of m/0h for "all all ... all" seed.
        PUBLIC_KEY = b"\x03\x0f\xdf^(\x9bZ\xefSb\x90\x95:\xe8\x1c\xe6\x0e\x84\x1f\xf9V\xf3f\xac\x12?\xa6\x9d\xb3\xc7\x9f!\xb0"
    else:
        PUBLIC_KEY = b""

    def __init__(
        self, msg: TxAckPaymentRequest, coin: coininfo.CoinInfo, keychain: Keychain
    ) -> None:
        from storage import cache
        from trezor.crypto.hashlib import sha256
        from trezor.utils import HashWriter

        from apps.common.address_mac import check_address_mac

        from .. import writers  # pylint: disable=import-outside-toplevel

        self.h_outputs = HashWriter(sha256())
        self.amount = 0
        self.expected_amount = msg.amount
        self.signature = msg.signature
        self.h_pr = HashWriter(sha256())

        if msg.nonce:
            nonce = bytes(msg.nonce)
            if cache.get(cache.APP_COMMON_NONCE) != nonce:
                raise DataError("Invalid nonce in payment request.")
            cache.delete(cache.APP_COMMON_NONCE)
        else:
            nonce = b""
            if msg.memos:
                DataError("Missing nonce in payment request.")

        writers.write_bytes_fixed(self.h_pr, b"SL\x00\x24", 4)
        writers.write_bytes_prefixed(self.h_pr, nonce)
        writers.write_bytes_prefixed(self.h_pr, msg.recipient_name.encode())
        writers.write_compact_size(self.h_pr, len(msg.memos))
        for m in msg.memos:
            if m.text_memo is not None:
                memo = m.text_memo
                writers.write_uint32(self.h_pr, _MEMO_TYPE_TEXT)
                writers.write_bytes_prefixed(self.h_pr, memo.text.encode())
            elif m.refund_memo is not None:
                memo = m.refund_memo
                # Unlike in a coin purchase memo, the coin type is implied by the payment request.
                check_address_mac(memo.address, memo.mac, coin.slip44, keychain)
                writers.write_uint32(self.h_pr, _MEMO_TYPE_REFUND)
                writers.write_bytes_prefixed(self.h_pr, memo.address.encode())
            elif m.coin_purchase_memo is not None:
                memo = m.coin_purchase_memo
                check_address_mac(memo.address, memo.mac, memo.coin_type, keychain)
                writers.write_uint32(self.h_pr, _MEMO_TYPE_COIN_PURCHASE)
                writers.write_uint32(self.h_pr, memo.coin_type)
                writers.write_bytes_prefixed(self.h_pr, memo.amount.encode())
                writers.write_bytes_prefixed(self.h_pr, memo.address.encode())

        writers.write_uint32(self.h_pr, coin.slip44)

    def verify(self) -> None:
        from trezor.crypto.curve import secp256k1

        if self.expected_amount is not None and self.amount != self.expected_amount:
            raise DataError("Invalid amount in payment request.")

        hash_outputs = writers.get_tx_hash(self.h_outputs)
        writers.write_bytes_fixed(self.h_pr, hash_outputs, 32)

        if not secp256k1.verify(
            self.PUBLIC_KEY, self.signature, self.h_pr.get_digest()
        ):
            raise DataError("Invalid signature in payment request.")

    def _add_output(self, txo: TxOutput) -> None:
        # For change outputs txo.address filled in by output_derive_script().
        assert txo.address is not None
        writers.write_uint64(self.h_outputs, txo.amount)
        writers.write_bytes_prefixed(self.h_outputs, txo.address.encode())

    def add_external_output(self, txo: TxOutput) -> None:
        self._add_output(txo)
        self.amount += txo.amount

    def add_change_output(self, txo: TxOutput) -> None:
        self._add_output(txo)


class Solution:
    def maxProduct(self, words: List[str]) -> int:
        ans = 0

        temp = []
        for i in words:
            temp.append(set(i))

        for i in range(1, len(words)):
            for j in range(i):
                if not temp[i].intersection(temp[j]):
                    ans = max(ans, len(words[i]) * len(words[j]))

        return ans


from django.urls import path

# importar as views que criamos na cadastros/views.py
from .views import CategoriaCreate, DespesaCreate
from .views import CategoriaUpdate, DespesaUpdate
from .views import CategoriaDelete, DespesaDelete
from .views import CategoriaList, DespesaList

urlpatterns = [
    # CADASTRAR
    path('cadastrar/categoria/', CategoriaCreate.as_view(),
         name='cadastrar-categoria'),
    path('cadastrar/despesa/', DespesaCreate.as_view(),
         name='cadastrar-despesa'),

    # EDITAR
    path('editar/categoria/<int:pk>/',
         CategoriaUpdate.as_view(), name='editar-categoria'),
    path('editar/despesa/<int:pk>/',
         DespesaUpdate.as_view(), name='editar-despesa'),

    # EXCLUIR
    path('excluir/categoria/<int:pk>/',
         CategoriaDelete.as_view(), name='excluir-categoria'),
    path('excluir/despesa/<int:pk>/',
         DespesaDelete.as_view(), name='excluir-despesa'),

    # LISTAR
    path('listar/categoria/', CategoriaList.as_view(), name='listar-categoria'),
    path('listar/despesa/', DespesaList.as_view(), name='listar-despesa'),
]


import json

from hashlib import md5
from typing import Dict, List, Optional, Tuple

from zeus.config import redis
from zeus.constants import Permission
from zeus.exceptions import ApiError, ApiUnauthorized, IdentityNeedsUpgrade
from zeus.models import Identity, Repository, User
from zeus.utils.github import GitHubClient
from zeus.utils.ssh import KeyPair

from .base import RepositoryProvider

ONE_DAY = 60 * 60 * 24

def get_github_client(user: User, scopes=()) -> Tuple[GitHubClient, Identity]:
    identity = Identity.query.filter(
        Identity.provider == "github", Identity.user_id == user.id
    ).first()

    if not identity:
        raise ApiUnauthorized

    for scope in scopes:
        if scope not in identity.scopes:
            raise IdentityNeedsUpgrade(scope=scope, identity=identity)

    return GitHubClient(token=identity.config["access_token"]), identity

class GitHubRepositoryProvider(RepositoryProvider):
    def get_owners(self, user: User) -> List[dict]:
        github, identity = get_github_client(user)
        response = github.get("/user/orgs")
        return [{"name": r["login"]} for r in response]

    def get_repos_for_owner(
        self, user: User, owner_name: str, include_private_repos=False
    ) -> List[dict]:
        if include_private_repos:
            github, identity = get_github_client(user, scopes=["repo"])
        else:
            github, identity = get_github_client(user)

        cache = GitHubCache(user=user, client=github, scopes=identity.scopes)

        results = []
        for repo_data in cache.get_repos(owner_name, no_cache=not self.cache):
            owner_name, repo_name = repo_data["full_name"].split("/", 1)
            results.append(
                {
                    "id": repo_data["id"],
                    "owner_name": owner_name,
                    "name": repo_name,
                    "permission": repo_data["permission"],
                    "url": repo_data["ssh_url"],
                    "config": {"full_name": repo_data["full_name"]},
                }
            )
        return results

    def get_repo(self, user: User, owner_name: str, repo_name: str) -> dict:
        github, identity = get_github_client(user)
        try:
            repo_data = github.get("/repos/{}/{}".format(owner_name, repo_name))
        except ApiError as exc:
            if exc.code == 404 and "repo" not in identity.scopes:
                raise IdentityNeedsUpgrade(scope="repo", identity=identity)

            raise

        owner_name, repo_name = repo_data["full_name"].split("/", 1)
        return {
            "id": repo_data["id"],
            "owner_name": owner_name,
            "name": repo_name,
            "url": repo_data["ssh_url"],
            "permission": Permission.admin
            if repo_data["permissions"].get("admin", False)
            else Permission.read,
            "config": {"full_name": repo_data["full_name"]},
        }

    def add_key(self, user: User, owner_name: str, repo_name: str, key: KeyPair):
        github, _ = get_github_client(user)
        github.post(
            "/repos/{}/{}/keys".format(owner_name, repo_name),
            json={"title": "zeus", "key": key.public_key, "read_only": True},
        )

    def get_permission(self, user: User, repo: Repository) -> Optional[bool]:
        try:
            repo = self.get_repo(user, *repo.data["full_name"].split("/", 1))
        except ApiError as exc:
            if exc.code == 404:
                return None

            raise

        return repo["permission"]

    def has_access(self, user: User, repo: Repository) -> bool:
        try:
            self.get_repo(user, *repo.data["full_name"].split("/", 1))
        except ApiError as exc:
            if exc.code == 404:
                return False

            raise

        return True

class GitHubCache(object):
    version = 4

    def __init__(self, user: User, client: GitHubClient = None, scopes=()):
        self.user = user
        self.scopes = scopes
        if client is None:
            self.client, _ = get_github_client(user)
        else:
            self.client = client

    def get_repos(self, owner: str, no_cache=False) -> List[Dict]:
        cache_key = "gh:{}:repos:{}:{}:{}".format(
            self.version,
            md5(self.client.token.encode("utf")).hexdigest(),
            md5(b",".join(s.encode("utf") for s in self.scopes)).hexdigest(),
            md5(owner.encode("utf-8")).hexdigest() if owner else "",
        )
        if no_cache:
            result = None
        else:
            result = redis.get(cache_key)

        if result is None:
            # TODO(dcramer): paginate
            if not owner:
                endpoint = "/user/repos"
                params = {"type": "owner"}
            else:
                endpoint = "/orgs/{}/repos".format(owner)
                params = {}
            result = []
            has_results = True
            while has_results and endpoint:
                response = self.client.get(endpoint, params=params)
                result.extend(
                    [
                        {
                            "id": r["id"],
                            "ssh_url": r["ssh_url"],
                            "full_name": r["full_name"],
                            "permission": Permission.admin
                            if r["permissions"].get("admin", False)
                            else Permission.read,
                        }
                        for r in response
                    ]
                )
                has_results = bool(response)
                if has_results:
                    endpoint = response.rel.get("next")
            redis.setex(cache_key, ONE_DAY, json.dumps(result))
        else:
            result = json.loads(result)
            for i in result:
                # we need to coerce permission back into our Permission enum
                i["permission"] = Permission(i["permission"])
        return sorted(result, key=lambda x: x["full_name"])


import MySQLdb
db = MySQLdb.connect('localhost','root','amber','unix')
cursor = db.cursor()
sql = """SELECT * FROM EMPLOYEE;"""
try:
	cursor.execute(sql)
	result = cursor.fetchall()
	for row in result:
		ID = row[0]
		NAME = row[1]
		SEX = row[2]
		AGE = row[3]
		print("ID = %d, NAME = %s, SEX = %s, AGE = %d" %(ID,NAME,SEX,AGE))
except:
	print("OOOPS SOMETHING IS WRONG")
db.close()


import argparse
import logging
import os
import sys

import numpy
import pandas

import gdp32datatools

logger = logging.getLogger(__name__)

TEM = """Created by {script_fn} from {input_fn}
  7  3                     !source type, field polarization
  {tx_x:8.2f} {tx_y:8.2f} {tx_z:1.0f} {rx_x:8.2f} {rx_y:8.2f} {rx_z:1.0f}   !tx and rx - x,y,z, truncated
  20 20                  !tx loop size
  3 3 4                    !in/out inversion
  3 1                      !tx waveform - userdefined, only one
  2 -7.8125e-3 -7.3125e-3 0.00 1.00 0.00e+0 2.0e-6 1.00 0.00 !tx waveform
  0 0 0                    !no front gate filter
"""

MOD = """Created by {script_fn} from {input_fn}
1 1  0    ! # of tem files, some vertical constraints yes, no topo
1 1 {TEMfn}
50    ! max # of iterations
{nlayers:6.0f}   !# of layers in model
"""

def mainfunc(avgfile, invpath, stnfn, layers):
    # Reference Lines 4-9
    txramp = 0.015          # ms
    rxramp = 0.002          # ms
    antialias = 0.001       # ms

    # Reference Lines 10-11
    ramptime = txramp

    # Ref lines 12-14
    area_tx = 400
    area_rx = 250

    assert os.path.isfile(avgfile)
    avg_dirname = os.path.dirname(avgfile)
    avg_basename = os.path.basename(avgfile)
    avg_pre, ext = os.path.splitext(avg_basename)
    if not os.path.isdir(invpath):
        os.makedirs(invpath)

    with open(avgfile, mode="r") as avgfobj:
        avg = gdp32datatools.read_avg1(avgfobj)
    stations = avg.Station.unique()

    stnlocs = None
    if stnfn:
        assert os.path.isfile(stnfn)
        stnlocs = pandas.read_csv(stnfn, delim_whitespace=True,
            names=["Station", "Easting", "Northing", "Elevation"],
            index_col="Station")
        logger.debug('Read station file %s' % stnfn)
    if stnlocs is None:
        stn = pandas.Series([0, 0, 0], index=["Easting", "Northing", "Elevation"])
        logger.debug('No station file, using coordinates 0, 0, 0')

    # Inversion setup. 1l through 7l, and a 20l
    for nlayers in [int(nl) for nl in layers]:
        linvpath = os.path.join(invpath, "%.0fl" % nlayers)
        if not os.path.isdir(linvpath):
            os.makedirs(linvpath)

        for station in stations:
            if not stnlocs is None:
                stn = stnlocs.loc[station]
            data = avg[avg.Station == station]

            fnprefix = "{avgpre}_{nlayers}l_{station}".format(
                avgpre=avg_pre, nlayers=nlayers, station=str(station))

            temfn = fnprefix + ".tem"
            temfnpath = os.path.join(linvpath, temfn)

            rtime = (numpy.asarray(data.Time) + ramptime) / 1e3  # Time after Tx turn-off
            norm_mag = numpy.asarray(data.Magnitude) / (area_rx * 1e6) # Magnitude in V/A not uV/A
            norm_pc_mag = numpy.asarray(data["%Mag"] / 100.)  # Zonge errors not relative.
            log_late_res = numpy.asarray(numpy.log10(
                numpy.power(area_rx * area_tx / data.Magnitude, 2/3.) *
                numpy.power(data.Time, -5/3.) *
                6.3219e-3
                ))
            norm_pc_mag[norm_pc_mag < 0.05] = 0.05
            data_lines = []
            skip = numpy.asarray(data.skp)
            for i in range(len(rtime)):
                skip_char = " "
                if skip[i] == 1:
                    skip_char = "%"
                # print station, i, rtime[i]
                data_lines.append(
                    "%s%4.3e %4.3e %3.2e 1 0" % (skip_char, rtime[i],
                        norm_mag[i], norm_pc_mag[i]))

            with open(temfnpath, mode="w") as temfobj:
                temfobj.write(str(TEM).format(
                    script_fn=__file__, input_fn=avgfile,
                    tx_x=stn.Easting, tx_y=stn.Northing, tx_z=stn.Elevation,
                    rx_x=stn.Easting, rx_y=stn.Northing, rx_z=stn.Elevation))
                temfobj.write("\n".join(data_lines))
                logger.info('Written TEM data file %s' % temfnpath)

            geomean_late_res = 10 ** numpy.mean(log_late_res)
            late_time = sorted(numpy.asarray(data.Time))[-1] / 1e3
            bos_depth2 = numpy.sqrt(
                late_time * geomean_late_res / numpy.pi * 2 * 4e-7)
            bos_depth = numpy.sqrt(
                (late_time * geomean_late_res) / (numpy.pi * 2 * 4e-7))
            log_step = numpy.log10(bos_depth) / nlayers

            # ================= Model file ===========================

            modfn = fnprefix + ".mod"
            modfnpath = os.path.join(linvpath, modfn)

            if nlayers != 20:   # ------- DISCRETE LAYER MODEL ---------------
                # Starting resistivities...
                mod_lines = [
                    "  {start_res:6.2f}   -1   0.6 !r {nlayer:2.0f}".format(
                        start_res=geomean_late_res, nlayer=n+1)
                    for n in range(nlayers)]

                if nlayers > 1:
                    # Starting thicknesses...
                    thks = numpy.empty((nlayers - 1))
                    thks[0] = 10 ** log_step
                    for i in range(1, len(thks)):
                        thks[i] = 10**(i * log_step) - 10**((i - 1) * log_step)
                    mod_lines += [
                        "  {start_thk:6.2f}   -1   9e9 !t {nlayer:2.0f}".format(
                            start_thk=thks[i], nlayer=i+1)
                        for i in range(len(thks))]

                    # Starting depths...
                    mod_lines += [
                        ("  -1 -1.000e+00     !z %2.0f" % (i + 1))
                        for i in range(len(thks))]

            elif nlayers == 20:     # ---------- SMOOTH MODEL ---------------
                # Starting resistivities
                mod_lines = [
                    "  {start_res:6.2f}   -1   .3 !r {nlayer:2.0f}".format(
                        start_res=geomean_late_res, nlayer=n+1)
                    for n in range(nlayers)]

                # Starting thicknesses...
                thks = numpy.empty((nlayers - 1))
                thks[0] = 10 ** log_step
                for i in range(1, len(thks)):
                    thks[i] = 10**(i * log_step) - 10**((i - 1) * log_step)
                mod_lines += [
                    "  {start_thk:6.2f}  1.000e-03 1e3  9e9 !t {nlayer:2.0f}".format(
                        start_thk=thks[i], nlayer=i+1)
                    for i in range(len(thks))]

                # Starting depths...
                mod_lines += [
                    ("  -1 -1.000e+00     !z %2.0f" % (i + 1))
                    for i in range(len(thks))]

            with open(modfnpath, mode="w") as modfobj:
                modfobj.write(str(MOD).format(
                    script_fn=__file__, input_fn=avgfile,
                    TEMfn=os.path.basename(temfnpath), nlayers=nlayers))
                modfobj.write("\n".join(mod_lines))
                logger.info('Written model file %s' % modfnpath)

def main():
    logging.basicConfig(
        level=logging.DEBUG,
        format=("%(asctime)s  %(levelname)s  %(filename)s  "
                "line %(lineno)d %(funcName)s : %(message)s"))
    p = argparse.ArgumentParser()
    p.add_argument('-i', '--invpath', default='.', help="Path to conduct inversions in")
    p.add_argument('-s', '--stnfn', default=None, help="Optional .stn file")
    p.add_argument('-l', '--layers', nargs='*', help='Layers e.g. 1 2 3 4 20', default=[1, 2, 3, 4, 5, 6, 7, 20])
    p.add_argument('avgfile', help="Required data in .avg file")
    args = p.parse_args(sys.argv[1:])
    return mainfunc(**args.__dict__)

if __name__ == "__main__":
    main()


# a = [1, 2, 3, 4, 5]
# b = [3, 1, 6, 4, 7]

# b, a= zip(*sorted(zip(b, a), reverse=False))

# print(a)
# print(b)

# 20 + ((x1**2 - 10 * np.cos(2*np.pi *x)) + (x2**2 - 10 * np.cos(2*np.pi *x2)))

arquivo = open("arquivo.txt", "w")
arquivo.write("N.Iteracoes \tSolucao\n")
num = 20
for i in range(100):
    num = num+2
    arquivo.write(str(i+1)+"\t\t\t\t"+str(num)+"\n")
arquivo.close


from unittest import TestSuite, makeSuite, main
import Testing
try:
    import Zope2
except ImportError: # BBB: for Zope 2.7
    import Zope as Zope2
Zope2.startup()

from Products.CMFCore.ActionInformation import ActionInformation
from Products.CMFCore.Expression import createExprContext
from Products.CMFCore.Expression import Expression
from Products.CMFCore.tests.base.dummy import DummyContent
from Products.CMFCore.tests.base.dummy import DummyTool as DummyMembershipTool
from Products.CMFCore.tests.base.testcase import SecurityTest

class ExpressionTests( SecurityTest ):

    def setUp( self ):

        SecurityTest.setUp(self)
        root = self.root
        root._setObject('portal', DummyContent('portal', url='url_portal'))
        portal = self.portal = root.portal
        self.folder = DummyContent('foo', url='url_foo')
        self.object = DummyContent('bar', url='url_bar')
        self.ai = ActionInformation(id='view'
                                  , title='View'
                                  , action=Expression(
                  text='view')
                                  , condition=Expression(
                  text='member')
                                  , category='global'
                                  , visible=1)

    def test_anonymous_ec(self):
        self.portal.portal_membership = DummyMembershipTool()
        ec = createExprContext(self.folder, self.portal, self.object)
        member = ec.global_vars['member']
        self.failIf(member)

    def test_authenticatedUser_ec(self):
        self.portal.portal_membership = DummyMembershipTool(anon=0)
        ec = createExprContext(self.folder, self.portal, self.object)
        member = ec.global_vars['member']
        self.assertEqual(member.getId(), 'dummy')

    def test_ec_context(self):
        self.portal.portal_membership = DummyMembershipTool()
        ec = createExprContext(self.folder, self.portal, self.object)
        object = ec.global_vars['object']
        portal = ec.global_vars['portal']
        folder = ec.global_vars['folder']
        self.failUnless(object)
        self.assertEqual(object.id, 'bar')
        self.assertEqual(object.absolute_url(), 'url_bar')
        self.failUnless(portal)
        self.assertEqual(portal.id, 'portal')
        self.assertEqual(portal.absolute_url(), 'url_portal')
        self.failUnless(folder)
        self.assertEqual(folder.id, 'foo')
        self.assertEqual(folder.absolute_url(), 'url_foo')

def test_suite():
    return TestSuite((
        makeSuite(ExpressionTests),
        ))

if __name__ == '__main__':
    main(defaultTest='test_suite')


from ampscz_asana.lib.server_scanner import grep_run_sheets
import re
import pandas as pd
from pathlib import Path
from ampscz_asana.lib.utils import convert_AU_to_US_date
from datetime import datetime
import os
import math
import json
from typing import Union

pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)

phoenix_dir = Path('/data/predict1/data_from_nda/Pronet/PHOENIX')
phoenix_dir = Path('/data/predict1/data_from_nda/Prescient/PHOENIX')

def get_entry_date_from_run_sheet(run_sheet: Path) -> str:
    ''''return entry date from the run sheet'''
    if 'Prescient' in str(run_sheet):
        df = pd.read_csv(run_sheet).T.reset_index()
        df.columns = ['field_name', 'field_value']
    else:
        df = pd.read_csv(run_sheet)

    entry_date = df.set_index('field_name').loc['chrmri_entry_date',
                                                'field_value']
    if pd.isna(entry_date):
        return ''

    if 'Prescient' in str(run_sheet):
        entry_date = convert_AU_to_US_date(entry_date)

    entry_date = re.sub('-', '_', entry_date)

    return entry_date

def get_mri_data(run_sheet: Path, entry_date: str) -> bool:
    ''''return the matching MRI zip file based on the entry_date

    Key argument:
        run_sheet: Path of the run sheet file used to get MRI folder, Path
        entry_date: date in YYYY_MM_DD format, str.

    Notes:
        The function will work regardless of zero padding in month and day of
        the entry_date, eg) 2000_03_03, 2000_3_3, 2000_03_3, and 2000_3_03
        will all be matched to 2022_03_03 pattern in the file name.
    '''
    if entry_date == '':
        return None

    # exact match
    for zip_file in run_sheet.parent.glob(f'*{entry_date}*.[Zz][Ii][Pp]'):
        return zip_file

    # date match
    for zip_file in run_sheet.parent.glob('*.[Zz][Ii][Pp]'):
        zip_filename_pattern = r'[A-Z]{2}\d{5}_MR_(\d{4}_\d{1,2}_\d{1,2})_'
        matching_pattern = re.search(zip_filename_pattern, zip_file.name)
        if matching_pattern:
            date_from_filename_str = matching_pattern.group(1)
            date_from_filename_date = datetime.strptime(date_from_filename_str,
                                                        '%Y_%m_%d')
            entry_date_date = datetime.strptime(entry_date, '%Y_%m_%d')

            if date_from_filename_date == entry_date_date:
                return zip_file

    return None

def check_mri_data(run_sheet: Path, entry_date: str) -> bool:
    ''''return if there is MR data and entry date

    Key argument:
        run_sheet: Path of the run sheet file, Path
        entry_date: date in YYYY_MM_DD format, str.

    Notes:
        The function will work regardless of zero padding in month and day of
        the entry_date, eg) 2000_03_03, 2000_3_3, 2000_03_3, and 2000_3_03
        will all be matched to 2022_03_03 pattern in the file name.
    '''
    if entry_date == '':
        return False

    # exact match
    for zip_file in run_sheet.parent.glob(f'*{entry_date}*.[Zz][Ii][Pp]'):
        return True

    # date match
    for zip_file in run_sheet.parent.glob('*.[Zz][Ii][Pp]'):
        zip_filename_pattern = r'[A-Z]{2}\d{5}_MR_(\d{4}_\d{1,2}_\d{1,2})_'
        matching_pattern = re.search(zip_filename_pattern, zip_file.name)
        if matching_pattern:
            date_from_filename_str = matching_pattern.group(1)
            date_from_filename_date = datetime.strptime(date_from_filename_str,
                                                        '%Y_%m_%d')
            entry_date_date = datetime.strptime(entry_date, '%Y_%m_%d')

            if date_from_filename_date == entry_date_date:
                return True

    return False

def check_when_transferred(expected_mri_path: Union[Path, str]) -> bool:
    ''''return the ctime of a file'''
    ctime = Path(expected_mri_path).stat().st_ctime
    date_str = datetime.fromtimestamp(ctime).strftime('%Y-%m-%d')
    return date_str

def is_qqc_executed(subject, entry_date) -> bool:
    if entry_date == '' or pd.isna(entry_date):
        return False

    mri_root = Path('/data/predict1/data_from_nda/MRI_ROOT')
    source_root = mri_root / 'sourcedata'

    date_numbers_only = re.sub('[-_]', '', entry_date)
    subject_dir = source_root / subject
    qqc_first_outputs = subject_dir.glob(f'*{date_numbers_only}*')

    if len(list(qqc_first_outputs)) >= 1:
        return True
    else:
        return False

def date_of_zip(subject, entry_date, phoenix_dir):
    if entry_date == '' or pd.isna(entry_date):
        return None
    formatted_entry_date = entry_date.replace("-", "_")
    formatted_entry_date = datetime.strptime(formatted_entry_date, '%Y_%m_%d')
    if 'Pronet' in phoenix_dir:
        prefix = 'Pronet'
    else:
        prefix = 'Prescient'
    base_dir = Path(f'/data/predict1/data_from_nda/{prefix}/PHOENIX/PROTECTED')
    pronet_dir = None
    for dir_name in os.listdir(base_dir):
        if dir_name.startswith(f'{prefix}{subject[:2]}'):
            pronet_dir = dir_name
            break
    zip_file_path = Path(base_dir, pronet_dir, 'raw', subject, 'mri')
    date_pattern = r'\d{4}_\d{1,2}_\d{1,2}'
    for filename in os.listdir(zip_file_path):
        date_match = re.search(date_pattern, filename)
        if date_match and entry_date != '':
            extracted_date = date_match.group(0)
            extracted_date = datetime.strptime(extracted_date, '%Y_%m_%d')
            if formatted_entry_date == extracted_date and \
                    filename[-4:] == '.zip' and 'MR' in filename:
                zip_file = zip_file_path / filename
                stat = zip_file.stat()
                timestamp = stat.st_mtime
                date_str = datetime.fromtimestamp(timestamp).strftime(
                        '%Y-%m-%d')
                return date_str
            else:
                continue

def date_of_qqc(subject, entry_date) -> str:
    if entry_date == '' or pd.isna(entry_date):
        return ''

    mri_root = Path('/data/predict1/data_from_nda/MRI_ROOT')
    source_root = mri_root / 'sourcedata'
    date_numbers_only = re.sub('[-_]', '', entry_date)
    subject_dir = source_root / subject
    qqc_first_outputs = list(subject_dir.glob(f'*{date_numbers_only}*'))
    if qqc_first_outputs:
        qqc_first_output = qqc_first_outputs[0]
        ctime = qqc_first_output.stat().st_ctime
        date_str = datetime.fromtimestamp(ctime).strftime('%Y-%m-%d')
        return date_str
    else:
        return ''

def is_mri_done(subject, entry_date) -> bool:
    if entry_date == '' or pd.isna(entry_date):
        return False
    mri_root = Path('/data/predict1/data_from_nda/MRI_ROOT')
    deriv_root = mri_root / 'derivatives' / 'mriqc'

    date_numbers_only = re.sub('[-_]', '', entry_date)
    subject_dir = deriv_root / f'sub-{subject}'
    mriqc_first_outputs = subject_dir.glob(f'*{date_numbers_only}*')

    if len(list(mriqc_first_outputs)) >= 1:
        return True
    else:
        return False

def is_fmriprep_done(subject, entry_date) -> bool:
    if entry_date == '' or pd.isna(entry_date):
        return False
    mri_root = Path('/data/predict1/data_from_nda/MRI_ROOT')
    deriv_root = mri_root / 'derivatives' / 'fmriprep'

    date_numbers_only = re.sub('[-_]', '', entry_date)
    subject_dir = deriv_root / f'sub-{subject}'
    mriqc_first_outputs = subject_dir.glob(f'*{date_numbers_only}*')

    if len(list(mriqc_first_outputs)) >= 1:
        return True
    else:
        return False

def is_dwipreproc_done(subject, entry_date) -> bool:
    if entry_date == '' or pd.isna(entry_date):
        return False
    mri_root = Path('/data/predict1/data_from_nda/MRI_ROOT')
    deriv_root = mri_root / 'derivatives' / 'dwipreproc'

    date_numbers_only = re.sub('[-_]', '', entry_date)
    subject_dir = deriv_root / f'sub-{subject}'
    mriqc_first_outputs = subject_dir.glob(f'*{date_numbers_only}*')

    if len(list(mriqc_first_outputs)) >= 1:
        return True
    else:
        return False

def extract_variable_information(row: dict, col: str,
                                 variable_name: str, excluded_values: list,
                                 value_list: list) -> list:
    """Extract info from the 'row' dictionary, which is from REDCap json.

    This function takes a specific row and column from the given json file as
    input. Each row is a dictionary and each column is a specific key in that
    dictionary. For a given subject's json file, there is a dictionary for
    each timepoint that contains the values for every variable within that
    timepoint. This function is used by the extract_missing_data_information
    function to loop through a given row (dictionary/timepoint) to search for
    the column (key/variable) that matches the variable name that it is
    searching for. It then creates a string that contains the date from that
    row, the timepoint from that row, and the value for the specific variable
    that was being searched for.

    Key Arguments:
        row: data from REDCap json, dict.
        col: key of the dictionary, str.
        variable_name: variable name, str.
        excluded_values: list of excluded values, list of str.
        value_list: list of values, list of str.
    """

    domain_type_dict = {'1': 'clinical measures',
                        '2': 'EEG',
                        '3': 'Neuroimaging',
                        '4': 'cognition',
                        '5': 'genetic and fluid biomarkers',
                        '6': 'digital biomarkers',
                        '7': 'speech sampling'}

    reason_dict = {
        'M1': 'Refusal - no reason provided',
        'M2': 'Refusal - reason: had an unpleasant experience last time',
        'M3': 'Refusal - reason: anxiety associated with the '
              'assessment domain',
        'M4': 'Refusal - reason: too much time commitment '
              '- AMP SCZ assessments',
        'M5': 'Refusal - reason: too much time commitment - other studies',
        'M6': 'No show',
        'M7': 'Not booked',
        'M8': 'Not applicable',
        'M9': 'Uncontrollable circumstance',
        'M10':  'Other reason'}

    value = row[col]
    date = row['chrmri_entry_date']

    if col == variable_name and row[col] not in excluded_values:
        if variable_name == 'missing_data_complete':
            if row[col] == '2':
                value = 'Complete'
            elif row[col] == '0':
                value = 'Incomplete'
        elif variable_name == 'chrmiss_domain':
            if row[col] == '1':
                value = 'Entire domain selected as missing'
        elif 'chrmiss_domain_type' in variable_name:
            #print(variable_name)
            for key, item in domain_type_dict.items():
                if key in col and row[col] == '1':
                    value = item
        elif 'chrmiss_domain_spec' in variable_name:
            for key, item in reason_dict.items():
                if key in row[col]:
                    value = item
        value_list.append(f"Timepoint: {row['redcap_event_name']} | "
                          f"Date: {date} | {value}")
    elif (variable_name == 'comment') and \
            (('mri' in col and 'comment' in col) and row[col] != '') or \
            (col == 'chrmri_missing' and row[col] != '') or \
            (col == 'chrmri_missing_spec' and row[col] != ''):
        if row[col] in row.values():
            if ('mri' in col and 'comment' in col and row[col] != ''):
                value_list.append(f"Timepoint: {row['redcap_event_name']} | "
                                  f"Date: {date} | {row[col]}")
    return value_list

def extract_missing_data_information(subject: str, phoenix_dir: str) -> list:
    """This function is given a specific subject and network as input. It then creates
    a path for the json file that is associated with that subject and network. For each variable
    that is being searched for in the json file, there is a list that will contain each value
    for that variable, along with its the date and timepoint."""

    if 'Pronet' in str(phoenix_dir):
        network = 'Pronet'
    else:
        network = 'Prescient'

    site = network + subject[:2]
    subject_dir = Path(phoenix_dir) / f'PROTECTED/{site}/raw/{subject}'
    json_path = subject_dir / 'surveys' / f'{subject}.{network}.json'

    if json_path.exists():
        with open(json_path, 'r') as f:
            json_data = json.load(f)
    else:
        return ['Json not found', 'Json not found', 'Json not found',
                'Json not found', 'Json not found']

    comments_list = []
    missing_data_form_complete_list = []  # missing_data_complete
    domain_missing_list = []  # chrmiss_domain
    reason_for_missing_data_list = []  # chrmiss_domain_spec
    domain_type_missing_list = []  # chrmiss_domain_type
    variables = []

    for x in range(1, 8):
        variables.append('chrmiss_domain_type' + f'___{x}')

    variable_dict = {
            'variables': [
                {'missing_data_complete': ['']},
                {'chrmiss_domain': ['']},
                {'chrmiss_domain_spec': ['']}
                ]
            }
    for x in range(0, len(variables)):
        variable_dict['variables'].append({variables[x]: ['', '0']})

    value_lists = [missing_data_form_complete_list,
                   domain_missing_list,
                   reason_for_missing_data_list]

    for row in json_data:
        for col in row:
            comments_list = extract_variable_information(
                    row, col, 'comment', '', comments_list)
            for x in range(0, len(variable_dict['variables'])):
                for key, item in variable_dict['variables'][x].items():
                    if 'miss_domain_type' not in key:
                        value_lists[x] = extract_variable_information(
                                row, col, key, item, value_lists[x])
                    else:
                        domain_type_missing_list = \
                                extract_variable_information(
                                        row, col, key,
                                        item, domain_type_missing_list)
    list_of_lists = [domain_type_missing_list,
                     reason_for_missing_data_list,
                     domain_missing_list,
                     missing_data_form_complete_list,
                     comments_list]

    for x in range(0, len(list_of_lists)):
        list_of_lists[x] = list(set(list_of_lists[x]))
        list_of_lists[x] = re.sub(r'[\[\]]|\r?\n|\\N\\\\A|\n', '', str(list_of_lists[x]))
        list_of_lists[x] = re.sub(r'\s+', ' ', list_of_lists[x])

    return list_of_lists

def extract_mri_comments(run_sheet: Path) -> str:
    '''Extract comments from MRI run sheet'''

    if 'Prescient' in str(run_sheet):
        df = pd.read_csv(run_sheet).T.reset_index()
        df.columns = ['field_name', 'field_value']
    else:
        df = pd.read_csv(run_sheet)

    comment_df = df[df['field_name'].str.contains('_comment')]
    comment_df = comment_df[~comment_df['field_value'].isnull()]
    comment_df = comment_df[comment_df['field_value'] != -3]

    text = ''
    for comment, table in comment_df.groupby('field_value'):
        text += f'{comment} :'
        text += ', '.join(table['field_name'].to_list()) + '\n'

    return text

def extract_session_num(run_sheet: Path) -> str:
    '''Extract session number from MRI run sheet'''

    if 'Prescient' in str(run_sheet):
        df = pd.read_csv(run_sheet).T.reset_index()
        df.columns = ['field_name', 'field_value']
    else:
        df = pd.read_csv(run_sheet)

    session_index = df[df['field_name'] == 'chrmri_session_num'].index
    session_str = df.loc[session_index]['field_value']

    return session_str
    try:
        int(session_str)
    except ValueError:
        return None

def extract_missing_data_info_new(subject: str,
                                  phoenix_dir: str,
                                  scan_date: str,
                                  timepoint: Union['1', '2']) -> tuple:
    '''Extract missing info and timepoint from REDCap'''
    if 'Pronet' in str(phoenix_dir):
        network = 'Pronet'
    else:
        network = 'Prescient'

    site = network + subject[:2]
    subject_dir = Path(phoenix_dir) / f'PROTECTED/{site}/raw/{subject}'
    json_path = subject_dir / 'surveys' / f'{subject}.{network}.json'

    if json_path.exists():
        with open(json_path, 'r') as f:
            json_data = json.load(f)
    else:
        return None

    # load json file into json
    df = pd.DataFrame(json_data)

    # column names to extract
    domain_type_cols = [x for x in df.columns
                        if re.search(r'chrmiss_domain_type___3', x)]
    miss_time_cols = [x for x in df.columns if 'chrmiss_time' == x]
    miss_withdrawn_cols = [x for x in df.columns if 'chrmiss_withdrawn' == x]
    miss_discon_cols = [x for x in df.columns if 'chrmiss_discon' == x]

    if len(domain_type_cols) == 0 and len(miss_time_cols) == 0:
        return None

    # select columns in interest
    df = df[['redcap_event_name', 'chrmri_entry_date',
             'chrmri_missing', 'chrmiss_domain_spec'] +
            domain_type_cols + miss_time_cols +
            miss_withdrawn_cols + miss_discon_cols]

    if scan_date == '' or pd.isna(scan_date):
        timepoint_to_index_dict = {'1': 'baseline',
                                   '2': 'month_2'}
        timepoint_str = timepoint_to_index_dict[timepoint]
        try:
            scan_date_index = df[
                    df.redcap_event_name.str.contains(timepoint_str)].index[0]
        except IndexError:
            return None
    else:
        # only leave the event where there is matching chrmri_entry_date
        scan_date = datetime.strptime(scan_date,
                                      '%Y_%m_%d').strftime('%Y-%m-%d')

        # [0] added at the end because it returns list of a singe item
        try:
            scan_date_index = df[df.chrmri_entry_date == scan_date].index[0]
        except IndexError:
            return None

    timepoint = df.loc[scan_date_index]['redcap_event_name']
    mri_rs_missing_info = df.loc[scan_date_index]['chrmri_missing']
    missing_info = df.loc[scan_date_index]['chrmiss_domain_type___3']

    if 'chrmiss_time' in df.columns:
        miss_time = df.loc[scan_date_index]['chrmiss_time']
    else:
        miss_time = None

    if 'chrmiss_withdrawn' in df.columns:
        miss_withdrawn = df.loc[scan_date_index]['chrmiss_withdrawn']
    else:
        miss_withdrawn = None

    if 'chrmiss_discon' in df.columns:
        miss_discon = df.loc[scan_date_index]['chrmiss_discon']
    else:
        miss_discon = None

    return (missing_info, mri_rs_missing_info, timepoint, miss_time,
            miss_withdrawn, miss_discon)

def compare_dates(df: pd.DataFrame) -> pd.DataFrame:
    """This function is used to match the variables that were found
    in the json files with the specific subject dates in the main
    pandas dataframe. If there are no variable dates that match the entry
    dates, the variables are removed from the dataframe."""

    df['entry_date'] = df['entry_date'].str.replace('_', '-')

    for index, row in df.iterrows():
        entry_date = row['entry_date']

        # if pd.isna(entry_date):
            # df.drop(index, inplace=True)
            # continue

        for col in ['domain_type_missing', 'reason_for_missing_data',
                    'domain_missing', 'missing_data_form_complete',
                    'comments']:
            string_list = row[col].split(",'")
            for i in range(len(string_list)):
                string_list[i] = string_list[i].replace('"', '').replace(
                        "'", '')
                if string_list[i].count('|') >= 2:
                    date_str = string_list[i].split('|')[1].strip()[6:]
                    date_str = date_str.replace('_', '-')
                    if date_str == '' or entry_date == '':
                        string_list[i] = ''
                    else:
                        d1 = datetime.strptime(date_str, '%Y-%m-%d')
                        d2 = datetime.strptime(entry_date, '%Y-%m-%d')
                        if abs((d1 - d2).days) > 10:
                            string_list[i] = ''

            string_list = [s for s in string_list if s != '']
            row[col] = ','.join(string_list)
        df.loc[index] = row

    return df

def format_days(day_amount: int) -> str:
    '''Get 'day' or 'days' appended to the day_amount'''
    if isinstance(day_amount, float) and not math.isnan(day_amount):
        day_amount = int(day_amount)
        if day_amount == 1:
            day_amount = str(day_amount) + ' day'
        else:
            day_amount = str(day_amount) + ' days'

    return day_amount

def get_run_sheet_df(phoenix_dir: Path,
                     datatype: str = 'mri',
                     test: bool = False,
                     subject: str = '*') -> pd.DataFrame:
    '''Summarize the raw data files based on the lochness created run sheets

    Key Arguments:
        phoenix_dir: Root of a PHOENIX directory, Path.
        datatype: data type, eg) 'mri', str.
    '''
    # get all run sheets extracted from RPMS or REDCap by lochness from
    run_sheets = grep_run_sheets(phoenix_dir, test)

    # create dataframe
    df = pd.DataFrame({'file_path': run_sheets})
    df['file_loc'] = df.file_path.apply(lambda x: str(x))
    # add network
    df['network'] = df.file_loc.str.contains('Prescient').map(
            {True: 'Prescient',
             False: 'Pronet'})

    # YA08362.Pronet.Run_sheet_mri_2.csv
    df['run_sheet_num'] = df.file_path.apply(lambda x: x.name).str.extract(
            '[A-Z]{2}\d{5}\.P\w+\.Run_sheet_\w+_(\d).csv')
    df['subject'] = df.file_loc.str.extract('([A-Z]{2}\d{5})')
    # AB00001.Pronet.Run_sheet_mri_1.csv
    df['datatype'] = df.file_path.apply(lambda x: x.name.split('_')[2])
    df['other_files'] = df['file_loc'].apply(lambda x: [x for x in os.listdir(
        Path(x).parent.absolute()) if re.search(r'zip', x, re.IGNORECASE)])
    df['other_files'] = df['other_files'].apply(lambda x: ', '.join(x))

    # select given datatype
    datatype_index = df[df['datatype'] == datatype].index
    datatype_df = df.loc[datatype_index]

    # check datatype file
    datatype_df['entry_date'] = datatype_df.file_path.apply(
            get_entry_date_from_run_sheet)

    # extract comments from run sheet
    datatype_df['run_sheet_comment'] = datatype_df.file_path.apply(
            extract_mri_comments)

    # extract comments from run sheet
    datatype_df['session_num'] = datatype_df.file_path.apply(
            extract_session_num)

    # for each run sheet, return the matching zip file
    datatype_df['expected_mri_path'] = datatype_df.apply(lambda x:
            get_mri_data(x['file_path'], x['entry_date']), axis=1)

    datatype_df['mri_data_exist'] = datatype_df.apply(lambda x:
            check_mri_data(x['file_path'], x['entry_date']), axis=1)

    # index of run sheets with matching MRI file
    index_with_mri_data = datatype_df[
            datatype_df['mri_data_exist'] == True].index

    # estimate MRI zip file arrival date
    datatype_df.loc[index_with_mri_data, 'mri_arrival_date'] = \
            datatype_df.loc[index_with_mri_data].expected_mri_path.apply(
                check_when_transferred)

    # extract missing data information and timepoint from the full REDCap data
    datatype_df['vars'] = datatype_df.apply(
            lambda x: extract_missing_data_info_new(x['subject'],
                                                    phoenix_dir,
                                                    x['entry_date'],
                                                    x['run_sheet_num']),
            axis=1)

    datatype_df['missing_info'] = datatype_df.vars.str[0]
    datatype_df['mri_rs_missing_info'] = datatype_df.vars.str[1]
    datatype_df['timepoint_text'] = datatype_df.vars.str[2]
    datatype_df['missing_timepoint'] = datatype_df.vars.str[3]
    datatype_df['missing_withdrawn'] = datatype_df.vars.str[4]
    datatype_df['missing_discon'] = datatype_df.vars.str[5]
    datatype_df['missing_marked'] = (
            (datatype_df['missing_info'] == '1') |
            (datatype_df['mri_rs_missing_info'] == '1') |
            (datatype_df['missing_timepoint'] == '1') |
            (datatype_df['missing_withdrawn'] == '1') |
            (datatype_df['missing_discon'] == '1')).map({True: 1, False: None})
    datatype_df.drop('vars', axis=1, inplace=True)

    datatype_df['qqc_executed'] = datatype_df.apply(lambda x:
            is_qqc_executed(x['subject'], x['entry_date']), axis=1)

    datatype_df['mriqc_done'] = datatype_df.apply(lambda x:
            is_mri_done(x['subject'], x['entry_date']), axis=1)

    datatype_df['fmriprep_done'] = datatype_df.apply(lambda x:
            is_fmriprep_done(x['subject'], x['entry_date']), axis=1)

    datatype_df['dwipreproc_done'] = datatype_df.apply(lambda x:
            is_dwipreproc_done(x['subject'], x['entry_date']), axis=1)

    datatype_df['qqc_date'] = datatype_df.apply(lambda x:
            date_of_qqc(x['subject'], x['entry_date']), axis=1)

    datatype_df['zip_date'] = datatype_df.apply(
            lambda x, param: date_of_zip(x['subject'], x['entry_date'], param),
            axis=1, param=str(phoenix_dir))
    datatype_df['entry_date'] = datatype_df['entry_date'].str.replace('_', '-')

    datatype_df = datatype_df.reset_index()
    datatype_df['qqc_date'] = pd.to_datetime(datatype_df['qqc_date'])
    datatype_df['zip_date'] = pd.to_datetime(datatype_df['zip_date'],
                                             errors='ignore')
    datatype_df['entry_date'] = pd.to_datetime(datatype_df['entry_date'])

    # estimate time difference
    arrival_qqc_time = lambda x: abs((x['zip_date'] - x['qqc_date']).days)
    arrival_scan_time = lambda x: abs((x['zip_date'] - x['entry_date']).days)
    delay_time = lambda x: abs((datetime.today() - x['entry_date']).days)
    datatype_df['days_arrival_to_qqc'] = datatype_df.apply(arrival_qqc_time,
                                                           axis=1)
    datatype_df['days_scan_to_arrival'] = datatype_df.apply(arrival_scan_time,
                                                            axis=1)
    datatype_df['days_scan_to_today'] = datatype_df.apply(delay_time, axis=1)
    datatype_df.reset_index(drop=True,inplace=True)
    datatype_df.drop('index', axis=1, inplace=True)

    return datatype_df

def dataflow_dpdash(datatype_df: pd.DataFrame,
                    outdir: Path,
                    test: bool = False) -> None:
    '''Convert datatype_df to DPDash importable format and save as csv files'''
    # flush existing files
    if not test:
        for i in outdir.glob('*mridataflow-day*csv'):
            os.remove(i)

    # loop through each subject to build database
    all_df = pd.DataFrame()
    for subject, table in datatype_df.groupby('subject'):
        for num, (timepoint, t_table) in enumerate(
                table.sort_values('timepoint').groupby('run_sheet_num'), 1):
            row = t_table.iloc[0]

            df_tmp = pd.DataFrame({
                'day': [num],
                'reftime': '',
                'timeofday': '',
                'weekday': '',
                'subject_id': subject,
                'site': subject[:2],
                'network': row['network'],
                'timepoint': row['run_sheet_num'],
                'scan_date': row['entry_date'],
                'missing_info': row['missing_marked'],
                'quick_qc': int(row['qqc_executed']),
                'manual_qc': row['mriqc_val'],
                'data_at_dpacc': int(row['mri_data_exist']),
                'days_arrival_to_qqc': row['days_arrival_to_qqc'],
                'days_scan_to_arrival': row['days_scan_to_arrival'],
                'days_scan_to_today': row['days_scan_to_today']})

            all_df = pd.concat([all_df, df_tmp])

    all_df['scan_date_missing'] = all_df.scan_date.apply(lambda x: 1 if
            pd.isna(x) else 0)
    all_df['scan_date'] = all_df.scan_date.fillna('No Scan Date')

    if test:
        return all_df

    # save CSV files
    # combined
    filename = f'combined-AMPSCZ-mridataflow-day1to{len(all_df)}.csv'
    nodate_df = all_df[all_df.scan_date.isnull()]
    date_df = all_df[~all_df.scan_date.isnull()]
    date_df.sort_values(['data_at_dpacc', 'days_scan_to_today', 'scan_date'],
                        inplace=True)
    all_df = pd.concat([date_df, nodate_df])
    all_df['day'] = range(1, len(all_df)+1)
    all_df.to_csv(outdir / filename, index=False)

    # for each network
    for network, table in all_df.groupby('network'):
        filename = f'combined-{network.upper()}-' \
                   f'mridataflow-day1to{len(table)}.csv'
        table['day'] = range(1, len(table)+1)
        table.to_csv(outdir / filename, index=False)

    # for each site
    for site, table in all_df.groupby('site'):
        filename = f'combined-{site}-mridataflow-day1to{len(table)}.csv'
        table['day'] = range(1, len(table)+1)
        table.to_csv(outdir / filename, index=False)

    # for each subject
    for subject, table in all_df.groupby('subject_id'):
        site = subject[:2]
        filename = f'{site}-{subject}-mridataflow-day1to{len(table)}.csv'
        table['day'] = range(1, len(table)+1)
        table.to_csv(outdir / filename, index=False)


a = ('12', 'a15', '18', '21', '24')
b = ('16', '20', '24b', '28', '32')
c = ('25', '30', '35', '4c0', '45')
# progression 是1个tuple，内层有3个tuple
progression = (a, b, c)
numbers = '0123456789'

# 第一层读 tuple 中的 tuple
for p in progression:
    int_sum = 0
    # 第二层读 tuple 中的 string
    for i in p:
        print(i)
        # 假设 i 是可以加总的数值
        is_num = True
        # 第三层读 string 中的 string(char 字元)
        for j in i:
            # print('   ', j)
            # 只要出现非数字，将 is_num 改成 False
            if numbers.find(j) == -1:
                is_num = False
                break
        # 判断 is_num 是否为 True
        if is_num:
            int_sum += int(i)
    print('Sum:', int_sum)


import json
from collections import OrderedDict

def config_list_to_dict(config_list):
    config_dict = OrderedDict()
    for i in json.loads(config_list):
        k = i.get('menu', None)
        if k is not None:
            if i.get('config', None) is not None:
                config_dict[k] = i['config']
    return config_dict


import pygame, sys
import numpy as np
import numpy.random as rd
import time
import pickle
import pygame.font

def get_symmetric_states(board):
    states = [board]

    # Rotations
    for _ in range(3):
        board = np.rot90(board)
        states.append(board)

    # Reflections
    board = np.flip(board, axis=0)
    states.append(board)
    for _ in range(3):
        board = np.rot90(board)
        states.append(board)

    states = list(set([board_to_str(state) for state in states]))
    return states
def get_max_Q_fom_symmetric_states(Q, board):
    max_Q = max(Q.get(s, 0) for s in get_symmetric_states(board))
    return max_Q

def get_min_Q_fom_symmetric_states(Q, board):
    min_Q = min(Q.get(s, 0) for s in get_symmetric_states(board))
    return min_Q

def board_to_str(board):
    board_ = board.astype(int)
    str_arr = board_.astype(str)
    return ''.join(str_arr.flatten())

def take_action(current_board, action, player):
    b = current_board.copy()
    b[action[0], action[1]] = player
    return b

def draw_horizontal_lines():
    y_axis = 0
    for i in range(2):
        y_axis += int(HEIGHT/3)
        pygame.draw.line( screen, LINE_COLOR, (0, y_axis), (WIDTH, y_axis), LINE_WIDTH )

def draw_vertical_lines():
    x_axis = 0
    for i in range(2):
        x_axis += int(WIDTH/3)
        pygame.draw.line( screen, LINE_COLOR, (x_axis, 0), (x_axis, HEIGHT), LINE_WIDTH )

def draw_lines():

    draw_horizontal_lines()
    draw_vertical_lines()

def draw_figures():
    x_square, y_square = WIDTH/3, HEIGHT/3
    for row in range(BOARD_ROWS):
        for col in range(BOARD_COLS):
            if board[row, col] == 1:
                pygame.draw.circle(screen, CIRCLE_COLOR, (int(col*x_square+x_square/2), int(row*y_square+y_square/2)),
                                   CIRCLE_RADIUS, CIRCLE_WIDTH)
            elif board[row, col] == 2:
                pygame.draw.line(screen, CROSS_COLOR, (col*x_square+SPACE, row*y_square+y_square-SPACE),
                                 (col*x_square+x_square-SPACE, row*y_square+SPACE), CROSS_WIDTH)
                pygame.draw.line(screen, CROSS_COLOR, (col*x_square+SPACE, row*y_square+SPACE),
                                 (col*x_square+x_square-SPACE, row*y_square+y_square-SPACE), CROSS_WIDTH)

def mark_square(board, row, col, player):
    board[row, col] = player
    return board

def initiate_player(human_player, AI_player):
    if human_player == 1:
        return human_player # human player's turn
    else:
        return AI_player # AI's turn

def available_square(row, col):
    return board[row, col] == 0

def is_board_full():
    val = np.prod(board)
    return val > 0

def handle_human_event(player, human_player, board, human_turn):
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            sys.exit()

        if player == human_player:
            if event.type == pygame.MOUSEBUTTONDOWN and not game_over and human_turn:
                human_turn = False
                mouseX = event.pos[0]
                mouseY = event.pos[1]

                clicked_row = mouseY // int(HEIGHT/3)
                clicked_col = mouseX // int(WIDTH/3)

                if board[clicked_row, clicked_col] == 0:
                    board = mark_square(board, clicked_row, clicked_col, player)
                    draw_figures()
                    player = 3 - player

        # if event.type == pygame.KEYDOWN:
        #     if event.key == pygame.K_r:
        #         restart()
        #         game_over = False
    return board, player, human_turn

def ai_max(available_actions, board, player, Q):
    possible_boards = [take_action(board, action, player) for action in available_actions]
    q_values = [Q.get(board_to_str(b), 0) for b in possible_boards]
    if np.all(board==0):
        rand_id = rd.randint(len(available_actions))
        action = available_actions[rand_id]
    else:
        action = available_actions[np.argmax(q_values)]
    clicked_row, clicked_col = action[0], action[1]
    return clicked_row, clicked_col

def ai_min(available_actions, board, player, Q):
    possible_boards = [take_action(board, action, player) for action in available_actions]
    q_values = [Q.get(board_to_str(b), 0) for b in possible_boards]
    if np.all(board==0):
        rand_id = rd.randint(len(available_actions))
        action = available_actions[rand_id]
    else:
        action = available_actions[np.argmin(q_values)]
    clicked_row, clicked_col = action[0], action[1]
    return clicked_row, clicked_col

def handle_ai_event(player, AI_player, Q, board):
    if player == AI_player:
        available_actions = np.argwhere(board == 0)
        if len(available_actions) > 0:
            if AI_player == 1:
                clicked_row, clicked_col = ai_max(available_actions, board, player, Q)
            else:
                clicked_row, clicked_col = ai_min(available_actions, board, player, Q)
            board = mark_square(board, clicked_row, clicked_col, player)
            draw_figures()
            player = 3 - player
    return board, player, True

def game_ended(board):
    for player in [1, 2]:
        for row_i in range(board.shape[0]):
            row = board[row_i]
            if np.all(row == player):
                draw_horizontal_winning_line(row_i, player)
                return True, player
        for col_i in range(board.shape[0]):
            col = board.T[col_i]
            if np.all(col == player):
                draw_vertical_winning_line(col_i, player)
                return True, player
        if np.all(np.diag(board) == player):
            draw_descending_diag(player)
            return True, player
        elif np.all(np.diag(np.fliplr(board)) == player):
            draw_ascending_diag(player)
            return True, player
    if np.all(board != 0):
        return True, 0  # Draw
    return False, None  # Game not ended

def draw_vertical_winning_line(col, player):
    x_square = WIDTH/3
    posX = col*x_square + (x_square/2)
    if player == 1:
        color = CIRCLE_COLOR
    else:
        color = CROSS_COLOR

    pygame.draw.line(screen, color, (posX, 15), (posX, HEIGHT-15), 10)

def draw_horizontal_winning_line(row, player):
    y_square = HEIGHT / 3
    posY = row * y_square + (y_square / 2)
    if player == 1:
        color = CIRCLE_COLOR
    else:
        color = CROSS_COLOR

    pygame.draw.line(screen, color, (15, posY), (WIDTH-15, posY), 10)

def draw_ascending_diag(player):
    if player == 1:
        color = CIRCLE_COLOR
    else:
        color = CROSS_COLOR
    pygame.draw.line(screen, color, (15, HEIGHT-15), (WIDTH-15, 15), 15)

def draw_descending_diag(player):
    if player == 1:
        color = CIRCLE_COLOR
    else:
        color = CROSS_COLOR
    pygame.draw.line(screen, color, (15, 15), (WIDTH-15, HEIGHT-15), 15)

def restart(player, board):
    screen.fill( BG_COLOR )
    draw_lines()
    player = 1
    board[:,:] = np.zeros((BOARD_ROWS, BOARD_COLS))
    return board, player

def start_screen():
    pygame.font.init()
    font_path = "04B_30__.TTF"
    font_size = 20
    font = pygame.font.Font(font_path, font_size)
    title_font = pygame.font.Font(font_path, font_size+10)
    note_font = pygame.font.Font(font_path, font_size - 5)
    title_surface = title_font.render("TIC TAC TOE", True, (0, 0, 0))
    title_rect = title_surface.get_rect(center=(WIDTH // 2, HEIGHT // 2 - 200))
    text_surface1 = font.render("Press '1' to be Player 1 (O)", True, (0, 0, 0))
    text_surface2 = font.render("Press '2' to be Player 2 (X)", True, (0, 0, 0))
    note_surface = note_font.render("Programmed by Caner Ates", True, (0, 0, 0))
    note_rect = note_surface.get_rect(center=(WIDTH // 2, HEIGHT // 2 + 100))
    screen.fill(BG_COLOR)
    screen.blit(title_surface, title_rect)
    screen.blit(text_surface1, (WIDTH // 2 - text_surface1.get_width() // 2, HEIGHT // 2 - 100))
    screen.blit(text_surface2, (WIDTH // 2 - text_surface2.get_width() // 2, HEIGHT // 2))
    screen.blit(note_surface, note_rect)
    pygame.display.update()
    #pygame.display.flip()

    while True:
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                sys.exit()
            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_1:
                    return 1
                elif event.key == pygame.K_2:
                    return 2

def end_screen(winner, human_player, AI_player, player, board):
    if winner == human_player:
        text = "You won!"
    elif winner == AI_player:
        text = "You lost!"
    else:
        text = "It's a draw!"
    pygame.font.init()
    font_path = "04B_30__.TTF"
    font_size1 = 40
    font_size2 = 20
    font1 = pygame.font.Font(font_path, font_size1)
    font2 = pygame.font.Font(font_path, font_size2)
    text_surface1 = font1.render(text, True, (0, 0, 0))
    text_surface2 = font2.render("Press 'R' to restart the game", True, (0, 0, 0))
    # screen.fill(BG_COLOR)
    screen.blit(text_surface1, (WIDTH // 2 - text_surface1.get_width() // 2, HEIGHT // 2 - 100))
    screen.blit(text_surface2, (WIDTH // 2 - text_surface2.get_width() // 2, HEIGHT // 2))
    pygame.display.update()
    #pygame.display.flip()

    while True:
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                sys.exit()
            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_r:
                    return restart(player, board)

pygame.init()

WIDTH = 600
HEIGHT = 600
LINE_WIDTH = 30
BOARD_ROWS = 3
BOARD_COLS = 3
CIRCLE_RADIUS = 65
CIRCLE_WIDTH = 20
CROSS_WIDTH = 30
SPACE = 55
# rgb: red green blue
RED = (255, 0 , 0)
BG_COLOR = (255, 204, 229)
LINE_COLOR = (216, 162, 162)
CIRCLE_COLOR = (0, 102, 0)
CROSS_COLOR = (66, 66, 66)

# board
board = np.zeros((BOARD_ROWS, BOARD_COLS))

# Load the dictionary from the file
with open('Q_table.pickle', 'rb') as f:
    Q = pickle.load(f)

len(Q)

screen = pygame.display.set_mode((WIDTH, HEIGHT))
pygame.display.set_caption('TIC TAC TOE')
screen.fill(BG_COLOR)

human_player = 1
AI_player = 2
player = initiate_player(human_player, AI_player)

game_over = False
start = False

human_player = start_screen()
AI_player = 3 - human_player
human_turn = False
if human_player == 1:
    human_turn = True
else:
    human_turn = False

screen.fill(BG_COLOR)
draw_lines()
# mainloop
while True:
        board, player, human_turn = handle_human_event(player, human_player, board, human_turn)
        pygame.display.update()
        ended, winner = game_ended(board)
        if ended:
            game_over = True
            board, player = end_screen(winner, human_player, AI_player, player, board)
            game_over = False

        board, player, human_turn = handle_ai_event(player, AI_player, Q, board)
        ended, winner = game_ended(board)
        if ended:
            game_over = True
            board, player = end_screen(winner, human_player, AI_player, player, board)
            game_over = False
        pygame.display.update()


import time
from selenium.webdriver.common.by import By
from pages.base_page import BasePage

class HomePage(BasePage):
    '''首页'''
    # 引导页页数
    welcome_pages = 3

    welcome_enter_button_locator = (By.ID, "com.xxzb.fenwoo:id/btn_start")

    def welcome(self):
        '''滑屏，点击欢迎页进入首页'''
        time.sleep(2)
        print('''首页''')
        for i in range(self.welcome_pages):
            print('滑屏开始')
            self.swipe_left()
            time.sleep(0.2)
        self.wait_click_element(self.welcome_enter_button_locator).click()


# Go fish!

import random
import os

class Player:
    def __init__(self, name, playerset, pairs):
        self.name = name
        self.playerset = playerset
        self.pairs = pairs

    def setName(self, name):
        self.name = name

    def getName(self):
        return self.name

def create_deck():
    Deck = []
    Kings = ["J", "Q", "K"]
    for i in range(4):
        for j in range(1, 11):
            Deck.append(j)
        for x in range(len(Kings)):
            Deck.append(Kings[x])
    return Deck

def sort_set(playerset):
    playerset = list(
        map(int, filter(lambda x: x.isdigit(), sorted(map(str, playerset))))
    ) + list(filter(lambda x: x.isalpha(), sorted(map(str, playerset))))
    return playerset

def check_pairs(player, playerset):
    try:
        for i in range(len(playerset)):
            if playerset[i] == playerset[i + 1] == playerset[i + 2] == playerset[i + 3]:
                player.playerset.remove(playerset[i])
                player.playerset.remove(playerset[i + 1])
                player.playerset.remove(playerset[i + 2])
                player.playerset.remove(playerset[i + 3])

                player.pairs += 1

    except Exception:
        return player.pairs

def print_winner(player1, player2):
    if player1.pairs > player2.pairs:
        print(player1.getName(), "won! He has", player1.pairs, "pairs.")
    elif player2.pairs < player1.pairs:
        print(player2.getName(), "won! He has", player2.pairs, "pairs.")
    else:
        print("It's a tie!")

def play_game():
    Deck = create_deck()

    random.shuffle(Deck)
    player1 = Player("", [], 0)
    player2 = Player("", [], 0)
    p1 = input("Player1, please enter your name: ")
    player1.setName(p1)
    p2 = input("Player2, please enter your name: ")
    player2.setName(p2)

    for i in range(5):
        player1.playerset.append(Deck[i])
        Deck.remove(Deck[i])

    for i in range(5):
        player2.playerset.append(Deck[i])
        Deck.remove(Deck[i])

    print("Initial deck: ", Deck)

    player = 1

    while len(Deck) > 0:
        os.system("cls")
        if player == 1:
            print()
            print("It's", player1.getName(), "'s turn!")
            print(player1.getName(), "set of cards: ", sort_set(player1.playerset))

            requiredCard = input("I need this card: ")
            try:
                r = int(requiredCard)
                if r in player2.playerset:
                    player1.playerset.append(r)
                    player2.playerset.remove(r)
                else:
                    print("Go fish!")
                    c = random.randint(0, len(Deck) - 1)
                    player1.playerset.append(Deck[c])
                    Deck.remove(Deck[c])

            except ValueError:
                if requiredCard in player2.playerset:
                    player1.playerset.append(requiredCard)
                    player2.playerset.remove(requiredCard)
                else:
                    print("Go fish!")
                    c = random.randint(0, len(Deck) - 1)
                    player1.playerset.append(Deck[c])
                    Deck.remove(Deck[c])

            player = 2
            check_pairs(player1, sort_set(player1.playerset))

            print(player1.getName(), "set of cards: ", sort_set(player1.playerset))
            print("Pairs of ", player1.getName(), ": ", player1.pairs)

        else:
            os.system("cls")
            print()
            print("It's", player2.getName(), "'s turn!")
            print(player2.getName(), "set of cards: ", sort_set(player2.playerset))
            requiredCard = input("I need this card: ")
            try:
                r = int(requiredCard)
                if r in player1.playerset:
                    player2.playerset.append(r)
                    player1.playerset.remove(r)
                else:
                    print("Go fish!")
                    c = random.randint(0, len(Deck) - 1)
                    player2.playerset.append(Deck[c])
                    Deck.remove(Deck[c])

            except ValueError:
                if requiredCard in player1.playerset:
                    player2.playerset.append(requiredCard)
                    player1.playerset.remove(requiredCard)
                else:
                    print("Go fish!")
                    c = random.randint(0, len(Deck) - 1)
                    player2.playerset.append(Deck[c])
                    Deck.remove(Deck[c])

            player = 1
            check_pairs(player2, sort_set(player2.playerset))
            print(player2.getName(), "set of cards: ", sort_set(player2.playerset))
            print("Pairs of ", player2.getName(), ": ", player2.pairs)
    print_winner(player1, player2)

play_game()


#  Escrever um programa que leia duas matrizes (denominadas A e B) do tipo vetor com 20 elementos reais. Construir uma matriz C, sendo cada elemento da matriz C a subtração de um elemento correspondente da matriz A com um elemento correspondente da matriz B, ou seja, a operação de processamento deve estar baseada na operação C[I] +- A[I] – B[1]. Ao final, apresentar os elementos da matriz C.
A = []
B = []
C = []
for i in range(0, 20):
  A.append(int(input('Digite {}° valor do vetor A: '.format(i + 1))))
for y  in range(0, 20):
  B.append(int(input('Digite {}° valor do vetor B: '.format(y + 1))))
for x in range(len(B)):
  C.append(A[x] - B[x])
for z in range(len(B)):
  print(C[z])


with open("input.txt") as f:
    data = f.read().splitlines()

def convertToNum(x):
    if x >= 'a' and x <= 'z':
        return ord(x)-ord('a') +1
    else:
        return ord(x)-ord('A') + 27

def part1(data):
    sum = 0
    for x in data:
        a = set(x[:len(x)//2]).intersection(set(x[len(x)//2:]))
        a = a.pop()#set to char
        sum += convertToNum(a)
    return sum

def part2(data):
    sum = 0
    for x in data:
        a =set(x[0]).intersection(set(x[1])).intersection(set(x[2]))
        a = a.pop()
        sum += convertToNum(a)
    return(sum)

print(f"part 1: {part1(data)}")
print(f"part 2: {part2(zip(*[iter(data)]*3))}")


import sys
import os
import pandas as pd
import numpy as np
from sklearn import metrics
from sklearn.ensemble import RandomForestRegressor
import pickle

train_datafile, test_datafile = sys.argv[1],sys.argv[2] if len(sys.argv) > 2 else print(" No input for train and test files")

train_df = pd.read_csv(train_datafile,sep=",")
test_df = pd.read_csv(test_datafile)

X_train = train_df.iloc[:,0:-1].values
y_train= np.array(train_df.iloc[:,-1])

X_test = test_df.iloc[:,0:-1].values
y_test = np.array(test_df.iloc[:,-1])

# Parameters
n_estimators = 5
random_state = 100
min_samples_leaf = 5
regressor = RandomForestRegressor(n_estimators = n_estimators, random_state = random_state, min_samples_leaf=min_samples_leaf)

regressor.fit(X_train,y_train)

y_pred = regressor.predict(X_test)

# Model evaluation
mae =  metrics.mean_absolute_error(y_test, y_pred)
mse = metrics.mean_squared_error(y_test, y_pred)
rmse = np.sqrt(metrics.mean_squared_error(y_test, y_pred))

# Save model
project_path = os.getcwd()
filename = project_path+"/model/housing_model.sav"

with open(filename,'wb') as model_file:
    pickle.dump(regressor, model_file)


from pyspark import SparkConf
from pyspark.sql import SparkSession
from pyspark.sql.types import ArrayType, StructField, StructType, StringType, IntegerType, DateType, FloatType
from pyspark.sql.functions import pandas_udf, PandasUDFType
import pandas as pd
from time import time
import numpy as np
from backtest_pairs.process_pairs import calculate_coint_results

## Use the following guide to setup an AWS EMR cluster
#https://towardsdatascience.com/getting-started-with-pyspark-on-amazon-emr-c85154b6b921

master = 'local[4]'
csv_path = "C:\\Users\\hamzajuzer\\Documents\\Algorithmic Trading\AlgoTradingv1\\backtest_pairs\\pyspark\\data\\etf_tickers_test.csv"
save_path = "C:\\Users\\hamzajuzer\\Documents\\Algorithmic Trading\AlgoTradingv1\\backtest_pairs\\pyspark\\data\\etf_tickers_test_results.csv"
min_period_yrs = 1.5
max_half_life = 30 # in time interval units
min_half_life = 2 # in time interval units
time_interval = 'daily'

if __name__ == '__main__':

    # Create Spark session
    conf = SparkConf().setMaster(master)
    spark = SparkSession.builder.config(conf=conf) \
        .getOrCreate()

    # Enable Arrow optimization and fallback if there is no Arrow installed
    spark.conf.set("spark.sql.execution.arrow.pyspark.enabled", "true")
    spark.conf.set("spark.sql.execution.arrow.pyspark.fallback.enabled", "true")

    # Read the dataframe from a csv
    schema = StructType([
        StructField('formatted_date', DateType(), True),
        StructField('x_0', FloatType(), True),
        StructField('x_1', FloatType(), True),
        StructField('combination', StringType(), True)
    ])

    df = spark.read.csv(csv_path, header=True, schema=schema, dateFormat="dd/MM/yyyy")
    df = df.withColumn('formatted_date',func.to_timestamp(func.col('formatted_date'), "yyyy-MM-dd"))

    schema_output = StructType([
        StructField('ticker', ArrayType(StringType(), True), True),
        StructField('max_date', DateType(), True),
        StructField('min_date', DateType(), True),
        StructField('n_samples', IntegerType(), True),
        StructField('johansen_90_p_trace', FloatType(), True),
        StructField('johansen_95_p_trace', FloatType(), True),
        StructField('johansen_99_p_trace', FloatType(), True),
        StructField('johansen_trace_stat', FloatType(), True),
        StructField('johansen_90_p_eigen', FloatType(), True),
        StructField('johansen_95_p_eigen', FloatType(), True),
        StructField('johansen_99_p_eigen', FloatType(), True),
        StructField('johansen_eigen_stat', FloatType(), True),
        StructField('johansen_eigenvectors', ArrayType(FloatType(), True), True),
        StructField('johansen_eigenvalue', FloatType(), True),
        StructField('adf_test_stat', FloatType(), True),
        StructField('adf_99_p_stat', FloatType(), True),
        StructField('adf_95_p_stat', FloatType(), True),
        StructField('adf_90_p_stat', FloatType(), True),
        StructField('hurst_exp', FloatType(), True),
        StructField('half_life_', FloatType(), True),
        StructField('sample_pass', IntegerType(), True),
        StructField('comment', StringType(), True)
    ])

    @pandas_udf(schema_output, PandasUDFType.GROUPED_MAP)
    def process_pairs_spark(df):

        ticker = df['combination'].iloc[0].split("_")
        ticker_col = [col for col in df if col.startswith('x')]

        merged_prices_comb_df = df.loc[:, df.columns != 'combination']
        merged_prices_comb_df.set_index('formatted_date', inplace=True)

        result_dict = calculate_coint_results(merged_prices_comb_df=merged_prices_comb_df,
                                              ticker=ticker,
                                              min_period_yrs=min_period_yrs,
                                              max_half_life=max_half_life,
                                              min_half_life=min_half_life,
                                              save_price_df=False,
                                              save_all=True,
                                              print_verbose=False,
                                              print_file=False,
                                              alt_cols=ticker_col,
                                              time_interval=time_interval)

        # return results dataframe
        results_df = pd.DataFrame()
        results_df = results_df.append(result_dict, ignore_index=True)
        return results_df

    t_0 = time()

    df_map = df.groupby("combination").apply(process_pairs_spark)
    df_map.show()
    results_df_p = df_map.toPandas()
    results_df_p.to_csv(save_path, mode='w', index=True)

    print(np.round(time() - t_0, 3), "seconds elapsed...")


#!/usr/bin/python3
# Author: Austin Cunningham
# Student Number: 20073379
# Version: 22/10/2016

import subprocess
#subprocess.call runs OS commands
# list all processes, search for mysql but exclude the grep in the response
output = subprocess.call('ps -A | grep mysql | grep -v grep', shell=True)
# 0 is a successfull response from the command and has found a the mysql process running
if output == 1 or output == 2:
    print ("MYSQL NOT RUNNING")
    print ("Do you want to start MySQL? y/n")
    output = input(":>")
    if output == 'y' or output == 'Y':
        # install mysql server and start services if prompted
        subprocess.call('sudo yum -y install mysql-server', shell=True)
        subprocess.call('sudo service mysqld start', shell=True)
        output2 = subprocess.call('ps -A | grep mysql | grep -v grep', shell=True)
        if output2 == 0:
            print("NOW STARTING MySQL SERVICE")
else:
    print ("MYSQL RUNNING")


import functools
import math

from ml.utils.logger import LOGGER
from ml.feature.sparse_vector import SparseVector
from ml.statistic import data_overview
import numpy as np
from ml.param.feature_binning_param import FeatureBinningParam
from computing.d_table import DTable

class Binning:
    """
    This is use for discrete data so that can transform data or use information for feature selection.

    Parameters
    ----------
    params : FeatureBinningParam object,
             Parameters that user set.

    Attributes
    ----------
    cols_dict: dict
        Record key, value pairs where key is cols' name, and value is cols' index. This is use for obtain correct
        data from a data_instance

    """

    def __init__(self, params: FeatureBinningParam, party_name: str, abnormal_list: list=None) -> None:
        self.params = params
        self.bin_num = params.bin_num
        self.cols_index = params.cols
        self.cols = []
        self.cols_dict = {}
        self.party_name = party_name
        self.header = None
        if abnormal_list is None:
            self.abnormal_list = []
        else:
            self.abnormal_list = abnormal_list
        self.iv_result = None
        self.splite_points = None

    def _init_cols(self, data_instances: DTable):

        # Already initialized
        if len(self.cols_dict) != 0:
            return

        header = data_overview.get_header(data_instances)
        self.header = header
        if self.cols_index == -1:
            self.cols = header
            self.cols_index = [i for i in range(len(header))]
        else:
            cols = []
            for idx in self.cols_index:
                try:
                    idx = int(idx)
                except:
                    raise ValueError("In binning module, selected index: {} is not integer".format(idx))

                if idx >= len(header):
                    raise ValueError(
                        "In binning module, selected index: {} exceed length of data dimension".format(idx))
                cols.append(header[idx])
            self.cols = cols

        self.cols_dict = {}
        for col in self.cols:
            col_index = header.index(col)
            self.cols_dict[col] = col_index

    def convert_feature_to_bin(self, data_instances, transform_cols_idx=-1, split_points=None):
        self._init_cols(data_instances)
        if transform_cols_idx is None:
            return data_instances, None, None

        if transform_cols_idx == -1:
            transform_cols_idx = self.cols_index
        else:
            assert isinstance(transform_cols_idx, (list, tuple))
            for col in transform_cols_idx:
                if col not in self.cols_index:
                    raise RuntimeError("Binning Transform cols: {} should be fit before transform".format(col))

        transform_cols_idx = list(map(int, transform_cols_idx))
        if split_points is None:
            split_points = self.split_points

        is_sparse = data_overview.is_sparse_data(data_instances)
        if is_sparse:
            f = functools.partial(self._convert_sparse_data,
                                  transform_cols_idx=transform_cols_idx,
                                  split_points_dict=split_points,
                                  header=self.header)
            new_data = data_instances.mapValues(f)
        else:
            f = functools.partial(self._convert_dense_data,
                                  transform_cols_idx=transform_cols_idx,
                                  split_points_dict=split_points,
                                  header=self.header)
            new_data = data_instances.mapValues(f)
        new_data.schema = {"header": self.header}
        bin_sparse = self.get_sparse_bin(transform_cols_idx, split_points)
        split_points_result = []
        for idx, col_name in enumerate(self.header):
            if col_name not in self.split_points:
                continue
            s_ps = self.split_points[col_name]
            s_ps = np.array(s_ps)
            split_points_result.append(s_ps)
        split_points_result = np.array(split_points_result)
        return new_data, split_points_result, bin_sparse

    @staticmethod
    def _convert_sparse_data(instances, transform_cols_idx, split_points_dict, header):
        all_data = instances.features.get_all_data()
        data_shape = instances.features.get_shape()
        indice = []
        sparse_value = []
        # print("In _convert_sparse_data, transform_cols_idx: {}, header: {}, split_points_dict: {}".format(
        #     transform_cols_idx, header, split_points_dict
        # ))
        for col_idx, col_value in all_data:
            if col_idx in transform_cols_idx:
                col_name = header[col_idx]
                split_points = split_points_dict[col_name]
                bin_num = Binning.get_bin_num(col_value, split_points)
                indice.append(col_idx)
                sparse_value.append(bin_num)
            else:
                indice.append(col_idx)
                sparse_value.append(col_value)

        sparse_vector = SparseVector(indice, sparse_value, data_shape)
        instances.features = sparse_vector
        return instances

    @staticmethod
    def _convert_dense_data(instances, transform_cols_idx, split_points_dict, header):
        features = instances.features
        for col_idx, col_value in enumerate(features):
            if col_idx in transform_cols_idx:
                col_name = header[col_idx]
                split_points = split_points_dict[col_name]
                bin_num = Binning.get_bin_num(col_value, split_points)
                features[col_idx] = bin_num

        instances.features = features
        return instances

    @staticmethod
    def get_bin_num(value, split_points):
        col_bin_num = len(split_points)
        for bin_num, split_point in enumerate(split_points):
            if value <= split_point:
                col_bin_num = bin_num
                break
        col_bin_num = int(col_bin_num)
        return col_bin_num

    def get_sparse_bin(self, transform_cols_idx, split_points_dict):
            """
            Get which bins the 0 located at for each column.
            """
            result = {}
            for col_idx in transform_cols_idx:
                col_name = self.header[col_idx]
                split_points = split_points_dict[col_name]
                sparse_bin_num = self.get_bin_num(0, split_points)
                result[col_idx] = sparse_bin_num
            return result


import math
from ctypes import *

in1, in2 = map(int, input().split())

def kek(a, b, c):
    c.value = "{}/{}".format(a // math.gcd(a, b), b // math.gcd(a, b))

k = c_wchar_p

kek(in1, in2, k)

print(k)


# -*- coding: utf-8 -*-
"""
Plotting the effect of variance on power flow time

@author: mikey
"""

import sys # import for utils
sys.path.append('../../')
sys.path.append('../')
import numpy as np
import time

from utils import import_zp
from montecarlo import generateMonteCarloBinaries, generateJson
from powerflowsim import PowerFlowSim

def calculateVariance(filename, length = 1000):
    sim = import_zp(filename)
    return np.var(sim['load']['profile'][0:length])

def viewLoadProfile(filename, length = 1000):
    import matplotlib.pyplot as plt
    sim = import_zp(filename)
    plt.plot(sim['load']['profile'][0:length])

def plotit(x, y):
    import matplotlib.pyplot as plt
    from textwrap import wrap
    fit = np.polyfit(x,y,1)
    fit_fn = np.poly1d(fit)
    plt.plot(x, y, 'yo', x, fit_fn(x), '--k')
    plt.legend(['Real', 'Regression'])

    plt.ylabel('Runtime (s)')
    plt.xlabel('Variance')
    title = 'Power Flow Simulation Runtime vs Load Variance for a 100 Node Radial Network'
    plt.title('\n'.join(wrap(title,50)))
    plt.savefig('pfs_simtime.pdf', transparent = True)
    plt.show()

no_houses = 100
generateJson(no_houses, 'mikework')
simlength = 100
data = [[], []]; k = 0
for i in np.arange(0.1, 10, 0.5):
    generateMonteCarloBinaries('C:/Users/mikey/Downloads/zp/18356', 0, no_houses, i, 'mikework')

    pfs = PowerFlowSim(simlength, 'radial', '../_configs/montecarlo' + str(no_houses) + '.json')
    before = time.time();
    pfs.nrPfSim()
    after = time.time();
    data[1].append(after-before);

    var = []
    for j in range(no_houses):
        var.append(calculateVariance('C:/Users/mikey/Downloads/zp/montecarlo' + str(j), simlength))
    data[0].append(np.mean(var));
    print('Runtime (s): ', data[1][k], ' Variance: ', data[0][k]); k += 1

plotit(data[0], data[1])


'''Crie um programa que leia nome, ano de nascimento e a carteira de trabalho e
cadastre-os(com idade) em um dicionário se por acaso a CTPS for diferente de
zero o dicionário receberá também o ano de contratação e o salário. Calcule e acrescente,
além da idade, com quantos anos a pessoa vai se aposentar'''
from datetime import date
atual = date.today().year

cadastro = dict()
cadastro['nome'] = str(input('Nome: '))
cadastro['ano-nasc'] = int(input('Ano Nascimento: '))
cadastro['carteira'] = int(input('Carteira de trabalho (0 = não tem): '))
cadastro['idade'] = atual - cadastro['ano-nasc']
if cadastro['carteira'] == 0:
    for k, v in cadastro.items():
        print(f'- {k} tem valor {v}')
else:
    cadastro['contrato'] = int(input('Ano Contratação: '))
    cadastro['salário'] = float(input('Salário: '))
    cadastro['aposenta'] = cadastro['idade'] + ((cadastro['contrato'] + 35) - atual)
    for k, v in cadastro.items():
        print(f' - {k} tem valor {v}')