#!/usr/bin/python
# -*- coding: utf-8 -*-

import os
import sys
from types import ModuleType

import ujson as json
from user_agents import parse

import django
from django.conf.urls import url
from django.core.wsgi import get_wsgi_application
from django.views.generic import View, TemplateView
from django.http import HttpResponse, JsonResponse, Http404
from django.forms import ModelForm, CharField
from django.views.decorators.http import require_POST, require_GET
from django.db.utils import IntegrityError
from django.apps.config import AppConfig

BASE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
sys.path.append(BASE_DIR)

import app
__package__ = 'app'

from .utils import clean_tags, USER_AGENT_HEADER, HTTP_HEADER_ENCODING
from .middlewares import basic_auth_handler, token_auth_handler

APP_LABEL = __package__

class App(AppConfig):
    verbose_name = 'Main'
    label = APP_LABEL

app = App('name', sys.modules[__name__])
FILE = __file__.split('.')[-2].split('/')[-1]

class Settings(ModuleType):
    DEBUG = os.environ.get('DEBUG', 'on') == 'on'
    APP_LABEL = APP_LABEL

    SECRET_KEY = os.environ.get('SECRET_KEY', os.urandom(32))

    ALLOWED_HOSTS = os.environ.get('ALLOWED_HOSTS', 'localhost').split(',')

    DATABASES = {
        'default': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': os.path.join(BASE_DIR, 'notes.db')}
    }
    ROOT_URLCONF = __name__
    MIGRATION_MODULES = {APP_LABEL: 'migrations'}
    INSTALLED_APPS = (app,)

    MIDDLEWARE_CLASSES = (
        APP_LABEL + '.' + FILE + '.TokenAuthentication',
        APP_LABEL + '.' + FILE + '.BasicAuthMiddleware',
        'django.middleware.security.SecurityMiddleware',
    )

# if "DJANGO_SETTINGS_MODULE" not in os.environ:
sys.modules['settings'] = Settings
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")

django.setup()

from .models import Note, Report, User, Token, NoteBook

OTHER = 'Other'

def get_limit():
    return 50

class NoteForm(ModelForm):
    _notebook = CharField(max_length=255, required=False)

    class Meta:
        model = Note
        fields = ['text', 'alias', '_notebook']

    def __init__(self, *args, **kwargs):
        super(NoteForm, self).__init__(*args, **kwargs)
        self.fields['alias'].required = False

class ReportForm(ModelForm):

    class Meta:
        model = Report
        fields = ['traceback']

def error(msg):
    return {'status': 'error', 'error': msg}

class NotesView(View):

    def get_queryset(self):
        if 'all' in self.request.GET:
            return self.request.user.notes.filter(active=True)
        if 'notebook' in self.request.GET:
            return self.request.user.notes.filter(active=True,
                                                  notebook__name=self.request.GET['notebook'])[:get_limit()]
        return self.request.user.notes.filter(active=True, notebook__isnull=True)[:get_limit()]

    def get(self, request, **kwargs):
        status = 200
        notes = self.get_queryset()
        response = [note.as_dict() for note in notes]
        if not response:
            response = error('No notes')
            status = 204
        return JsonResponse(response, status=status, safe=False)

    def post(self, request, **kwargs):
        status = 400
        response = error('No data')
        form = NoteForm(request.POST)
        if form.is_valid():
            status = 201
            data = {
                'text': form.cleaned_data['text'],
                '_notebook': form.cleaned_data.get('_notebook') or kwargs.get('name'),
                'alias': form.cleaned_data.get('alias', None),
                'owner': request.user
            }

            if not data['alias']:
                del data['alias']
            if data.get('_notebook'):
                data['notebook'] = NoteBook.get_or_create(data.pop('_notebook'))
            else:
                del data['_notebook']
            try:
                Note.objects.create(**data)
            except IntegrityError:
                status = 406
                response = error('Alias must be unique, use -o option to overwrite')
            else:
                response = {'status': 'ok'}
        return JsonResponse(response, status=status)

class NotebookView(NotesView):

    def get_queryset(self):
        qs = self.request.user.notes.filter(active=True, notebook__name=self.kwargs['name'])
        return qs[:get_limit()]

class NoteView(View):

    @property
    def note(self):
        alias = self.kwargs.get('alias')
        if not alias:
            raise Http404
        try:
            return self.request.user.notes.filter(alias=alias, active=True).get()
        except Note.DoesNotExist:
            raise Http404

    def get(self, request, **kwargs):
        response = self.note.as_dict()
        ua = parse(request.META.get(USER_AGENT_HEADER, ''))
        if ua.device.family != OTHER or ua.browser.family != OTHER:
            response = clean_tags(response['text'])
            ct = None
        else:
            response = json.dumps(response)
            ct = 'application/json'
        return HttpResponse(response, content_type=ct)

    def delete(self, *args, **kwargs):
        self.note.delete()
        return JsonResponse({'status': 'ok'}, status=204)

@require_POST
def report_view(request):
    status = 400
    response = error('No data')
    form = ReportForm(request.POST)
    if form.is_valid():
        report = form.instance
        if hasattr(request, 'user') and request.user:
            report.user = request.user
        report.info = request.META.get('HTTP_USER_AGENT', '')
        report.save()
        status = 201
        response = {'status': 'ok'}
    return JsonResponse(response, status=status)

@require_POST
def get_token(request):
    return JsonResponse({'status': 'ok', 'token': request.user.token.key}, status=201)

@require_POST
def drop_token(request):
    request.user.token.delete()
    return JsonResponse({'status': 'ok'}, status=202)

@require_GET
def get_install_script(request):
    with open(os.path.join(BASE_DIR, 'install.sh')) as script:
        return HttpResponse(script.read())

urlpatterns = [
    url(r'^notes/?$', NotesView.as_view()),
    url(r'^notes/(?P<alias>.{1,30})/?$', NoteView.as_view()),
    url(r'^notebook/(?P<name>.{1,30})/?$', NotebookView.as_view()),

    url(r'^report/?$', report_view, name='report'),
    url(r'^get_token/?$', get_token, name='get_token'),
    url(r'^drop_tokens/?$', drop_token, name='drop_token'),

    url(r'^install\.sh$', get_install_script),
]

application = get_wsgi_application()

class BasicAuthMiddleware:
    AUTH_HEADER = 'HTTP_AUTHORIZATION'

    @classmethod
    def get_authorization(cls, request):
        auth = request.META.get(cls.AUTH_HEADER, b'')
        if isinstance(auth, type('')):
            auth = auth.encode(HTTP_HEADER_ENCODING)
        return auth.split()

    @staticmethod
    def set_user(request, user):
        request.user = user

    @staticmethod
    def not_auth(realm='', header='WWW-Authenticate'):
        response = HttpResponse(status=401)
        response[header] = realm
        return response

    @classmethod
    def not_auth_base(cls, realm):
        realm = 'Basic realm="%s"' % realm
        return cls.not_auth(realm)

    def process_request(self, request):
        if hasattr(request, 'user'):
            return
        auth = self.get_authorization(request)
        request = basic_auth_handler(request, auth, self.not_auth_base, self.set_user, User)
        return request

class TokenAuthentication(BasicAuthMiddleware):

    @classmethod
    def not_auth_token(cls, realm=''):
        return cls.not_auth(realm, 'Token')

    def process_request(self, request):
        if hasattr(request, 'user'):
            return
        auth = self.get_authorization(request)
        request = token_auth_handler(request, auth, self.not_auth_token, self.set_user, Token)
        return request

if __name__ == "__main__":

    from django.core.management import execute_from_command_line
    execute_from_command_line(sys.argv)


# -*- coding: utf-8 -*-
"""
Created on Wed Nov  7 20:36:52 2018

@author: Matthew

creates multicolored hover over movable graph with bokeh,
make sure to change the syspath for autoguru and the model location on your run
"""

import numpy as np
import pandas as pd
from sklearn.manifold import TSNE
import bokeh.plotting as bk
from bokeh.models import HoverTool
from copy import deepcopy
import json
import random
from itertools import compress
import collections

import sys
# manually adds auto guru to my python path
sys.path.append(r'E:\hackathon\autoguru\question-answering')
from questionanswering.embeddings import Embedder
#change this for your program

def main():
    model = 'E:\\hackathon\\answer\\embedder.npz'
    # eembedder model location, change for yourprogram
    embedder = Embedder.load(model)

    with open('E://hackathon//final//question-answers.json') as f:
        pracDic = collections.OrderedDict(json.load(f))
    # load json dict
    embedded = np.zeros((len(pracDic),300))
    ind = 0
    question = []
    answers = []
    # initialize data structures
    for x in pracDic.items():
        question.append(x[0])
        answers.append(x[1])
        embedded[ind,:] = embedder.embed(x[0])
        ind += 1
    #construct structures from dictionary

    embedded = pd.DataFrame(embedded) # make embedded a dataframe

    # not needed, nothing should be null
    #pracDic = dict(compress(list(pracDic.items()),list(~embedded.isnull().any(axis=1))))
    #embedded = embedded.loc[~embedded.isnull().any(axis=1),:]

    tsne = TSNE(n_components =2,verbose=0,perplexity=12,n_iter=10000, early_exaggeration =15)
    tsne_results = np.array(tsne.fit_transform(embedded))
    # creates tsne model and fits our data to it
    finalQuest = pd.DataFrame({'quest':question,'ans':answers,'origVects':np.array(embedded).tolist(),
                               'vectX':tsne_results[:,0],'vectY':tsne_results[:,1]})
    df = deepcopy(finalQuest)
    # this isnt needed i didnt want to mess with finalQuest while debugging

    def splitFrame(df):
        ''' created a list of dataframes with each df having different questions'''
        ansSet = list(set(df['ans']))
        dfList = []
        for x in ansSet:
            manipDf = df.loc[df['ans'] == x,:]
            dfList.append(manipDf)
        return(dfList)
    splits = splitFrame(df)

    bk.output_file("toolbar.html")
    #sets the output url
    TOOLTIPS = '''

                Question:

                @quest{safe}

                Answer:

                @ans{safe}

'''
    #creates the hover over text

    p = bk.figure(plot_width=850, plot_height=700,title='Questions and Answers')
    p.title.text_color = 'black'
    p.title.text_font = 'helvetica'
    p.title.text_font_size = '24pt'
    p.background_fill_color = '#f4f3ef'
    p.xaxis.visible =False
    p.yaxis.visible = False
    p.xgrid.grid_line_color = None
    p.ygrid.grid_line_color = None
    #sets details for graph

    color = []
    for x in range(1000):
        color.append("#%06x" % random.randint(0, 0xFFFFFF))
    colors = list(set(color))
    #generates a list of colors

    # manually selecting colors, this is just for hte demo, comment out this line if using more
    #colors = ['firebrick','royalblue','peru','teal','seagreen','darkmagenta','gold','black','orange']

    for x in range(len(splits)):
        r = p.scatter('vectX','vectY', size=10,source=bk.ColumnDataSource(splits[x]),color=colors[x])
        p.add_tools(HoverTool(renderers=[r], tooltips=TOOLTIPS))
        #plots all of the different colored points with their respective hover text

    bk.show(p)
    return (splits,pracDic)
    #shows the plot
if __name__ == '__main__':
    (splits,pracDic) = main()


import datetime
from settings import *
from countries import country_dict
from forms import AvailabilityForm

class AthleteAvailability:
    def __init__(self, availability_form : AvailabilityForm) -> None:
        self.athlete_id = availability_form.athlete_id
        self.location_address = availability_form.location
        try:
            self.__convert_time(availability_form.dateTime_utc)
            self.__verify_country(availability_form.country)
            self.__item_expiry()
        except Exception as e:
            raise e

    def validate(self):
        pass

    def __convert_time(self, datetime_string):
        try:
            datetime_utc = datetime.datetime.fromisoformat(datetime_string)
            if ( (getattr(datetime_utc, 'minute', None) not in  [0, None] ) or (getattr(datetime_utc, 'second', None) not in  [0, None] ) or (getattr(datetime_utc, 'mircosecond', None) not in  [0, None] ) ):
                raise Exception(INVALID_TIME_RESOLUTION_GIVEN)

            if self.verify_datetime(datetime_utc) == False:
                raise Exception(TIME_TOO_EARLY)

            else:
                self.date = datetime_utc.date().isoformat()
                self.available_time = datetime_utc.time().isoformat()
        except Exception as e:
            raise e

    def __verify_country(self, country):
        if country in country_dict.keys():
            self.location_country = country_dict[country]
        else:
            raise Exception(INVALID_COUNTRY_GIVEN)

    def __item_expiry(self):
        availability_datetime = datetime.datetime.fromisoformat(f"{self.date} {self.available_time}")
        expiry_datetime = availability_datetime + datetime.timedelta(days=EXPIRY_IN_DAYS)
        self.expiry_datetime = expiry_datetime.isoformat()
        self.expiry_epoch = int(expiry_datetime.timestamp())

    def verify_datetime(self, datetime_utc):
        datetime_today = datetime.datetime.utcnow()
        if (datetime_utc < (datetime_today + datetime.timedelta(days=TIME_DELTA_WINDOW_IN_DAYS))):
            return False
        else:
            return True


# -*- coding: utf-8 -*-

'''
Given a string s1, we may represent it as a binary tree by partitioning it to two non-empty substrings recursively.

给定一个字符串 s1，我们可以将它表示成一个二叉树，方法是递归地将其划分为两个非空的子串。

Below is one possible representation of s1 = "great":

以下是对 s1='great'的一种表达方法：

    great
   /    \
  gr    eat
 / \    /  \
g   r  e   at
           / \
          a   t

To scramble the string, we may choose any non-leaf node and swap its two children.

为了打乱字符串，我们可以选择任意一个非叶子节点，交换他的两个子节点。

For example, if we choose the node "gr" and swap its two children, it produces a scrambled string "rgeat".

比如，如果我们选择节点"gr"并交换他的子节点，则他会产生一个乱序字符串"rgeat"。

    rgeat
   /    \
  rg    eat
 / \    /  \
r   g  e   at
           / \
          a   t
We say that "rgeat" is a scrambled string of "great".
Similarly, if we continue to swap the children of nodes "eat" and "at", it produces a scrambled string "rgtae".

我们称"rgeat"是"great"的一个乱序字符串。
同样地，如果我们继续交换"eat"和"at"的子节点，就可以得到乱序字符串"rgtae"。

    rgtae
   /    \
  rg    tae
 / \    /  \
r   g  ta  e
       / \
      t   a
We say that "rgtae" is a scrambled string of "great".
Given two strings s1 and s2 of the same length, determine if s2 is a scrambled string of s1.

我们称"rgtae"是"great"的一个乱序字符串。

给定两个同样长度的字符串 s1 和 s2，判断 s2 是否是 s1 的乱序字符串。

Example 1:
Input: s1 = "great", s2 = "rgeat"
Output: true
Example 2:
Input: s1 = "abcde", s2 = "caebd"
Output: false
'''

def isScramble(s1: str, s2: str) -> bool:
    '''
    使用递归，将s1的所有乱序字符串都列举出来，看是否包含s2.
    缺点：时间复杂度很大。T(n) = T(1)*T(n-1)*2 + T(2)*T(n-2)*2 + ... + T(n-1)*T(1)*2
    '''
    def get_scrambled(s):
        '''
        递归得到s的所有乱序字符串。
        '''
        n = len(s)
        if n <= 1:
            return [s]
        result_list = []
        for i in range(1, n): # 两部分：s[:i] 和 s[i:]
            group1 = get_scrambled(s[:i])
            group2 = get_scrambled(s[i:])
            # [:i] 和 s[i:] 的所有乱序字符串进行笛卡尔积，并且有前后之分
            for s1 in group1:
                for s2 in group2:
                    result_list.append(s1+s2)
                    result_list.append(s2+s1)
        return result_list
    return s2 in get_scrambled(s1)

def isScramble2(s1: str, s2: str) -> bool:
    """
    s1 与 s2 互为乱序字符串 <=> 存在一种分割，将 s1 和 s2 分别分割为两部分，两部分的大小分别对应相等，且两部分分别互为乱序字符串。
    """
    def fun(s1:str, s2:str, memo_dict:dict):
        """
        递归，判断 s1 和 s2 是否互为乱序字符串。
        """
        # 递归边界条件
        if len(s1) != len(s2):
            return False
        if len(s1) <= 1:
            return s1 == s2
        if (s1, s2) in memo_dict or (s2, s1) in memo_dict:
            return memo_dict.get((s1, s2), False) or memo_dict.get((s2, s1), False)
        # 寻找一种分割，使得两部分分别互为乱序字符串
        for i in range(1, len(s1)):
            if (fun(s1[:i], s2[:i], memo_dict) and fun(s1[i:], s2[i:], memo_dict)) or\
            (fun(s1[:i], s2[-i:], memo_dict) and fun(s1[i:], s2[:-i], memo_dict)):
                memo_dict[(s1, s2)] = True
                memo_dict[(s2, s1)] = True
                return True
        memo_dict[(s1, s2)] = False
        memo_dict[(s2, s1)] = False
        return False
    return fun(s1, s2, {})

if '__main__' == __name__:
    s1 = "great"
    s2 = "rgeat"
    print(isScramble2(s1, s2))


import psycopg2
import os
import sys
import datetime
from collections import Counter
from types import *
import argparse

from queries import *

correct_answers = []

parser = argparse.ArgumentParser()
parser.add_argument('-i', '--interactive', help="Run queries one at a time, and wait for user to proceed", required=False, action="store_true")
parser.add_argument('-q', '--query', type = int, help="Only run the given query number", required=False)
args = parser.parse_args()

interactive = args.interactive

conn = psycopg2.connect("dbname=elections user=vagrant")
cur = conn.cursor()

for i in range(0, 8):
    # If a query is specified by -q option, only do that one
    if args.query is None or args.query == i:
        try:
            if interactive:
                os.system('clear')
            print("========== Executing Query {}".format(i))
            print(queries[i])
            cur.execute(queries[i])

            if i in [1, 2, 3]:
                ans = cur.fetchall()
                correct_answers.append(ans)

                print("--------- Your Query Answer ---------")
                for t in ans:
                    print(t)
                print("")
            else:
                if i in [4]:
                    conn.commit()
                    query_string = "select name, statecode, num_counties(name) from states order by name limit 10"
                    print("--------- Running {} -------".format(query_string))
                    cur.execute(query_string)
                    ans = cur.fetchall()
                    print("-- Result")
                    for t in ans:
                        print(t)
                    print("")
                if i in [5]:
                    conn.commit()
                    query_string = "select statecode, name, presidential_winner(statecode, name, 2008) from counties where statecode = 'MD' order by statecode limit 10"
                    print("--------- Running {} -------".format(query_string))
                    cur.execute(query_string)
                    ans = cur.fetchall()
                    print("-- Result")
                    for t in ans:
                        print(t)
                    print("")
                if i in [6]:
                    conn.commit()
                    query_string = 'SELECT n.nspname as "Schema", p.proname as "Name", pg_catalog.pg_get_function_result(p.oid) as "Result data type" FROM pg_catalog.pg_proc p LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace WHERE p.proname = \'update_num_large_counties_on_insert\''
                    print("--------- Running {} -------".format(query_string))
                    cur.execute(query_string)
                    ans = cur.fetchall()
                    print("-- Result (should list the trigger function)")
                    for t in ans:
                        print(t)
                    print("")
                if i in [7]:
                    conn.commit()
                    query_string = "insert into counties values('Fake', 'MD', 0, 10000000)"
                    cur.execute(query_string)
                    conn.commit()
                    query_string = "insert into counties values('Fake', 'CA', 0, 10000000)"
                    cur.execute(query_string)
                    conn.commit()
                    query_string = "select * from num_large_counties"
                    print("--------- Running {} -------".format(query_string))
                    cur.execute(query_string)
                    ans = cur.fetchall()
                    print("-- Result (should list California with 10 and Maryland with 1)")
                    for t in ans:
                        print(t)
                    print("")

            if interactive:
                input('Press enter to proceed')
                os.system('clear')
        except:
            print(sys.exc_info())
            raise


def build(bld):
    bld.DefineGem(
        Editor = dict(
            vs_filter       = 'Gems/ScriptCanvas',
            win_features    = ['crcfix'],
            autod_uselib    = ['QT5CORE','QT5GUI','QT5WIDGETS'],
            uselib          = ['AWS_CPP_SDK_CORE'],
            features        = ['qt5','az_code_gen'],
            configurations  = ['profile', 'debug', 'debug_test', 'profile_test'],
            includes        = [ '.', '..', 'Source'],
            defines         = ['SCRIPTCANVAS_EDITOR', 'SCRIPTCANVAS', 'SCRIPTCANVAS_ERRORS_ENABLED', 'ENABLE_EXTENDED_MATH_SUPPORT=0'],
            use             = ['ScriptCanvas', 'ScriptCanvasEditor', 'AssetBuilderSDK', 'GraphCanvasWidgets', 'AzToolsFramework'],
            file_list       = ['scriptcanvastestingeditor.waf_files'],
            test_all_file_list = ['scriptcanvastestingeditor_tests.waf_files'],
            disable_pch     = True,
        ),
    )


import numpy as np
import os
import pytest
import random
import tensorflow as tf
import torch
import utils

__author__ = "Christopher Potts"
__version__ = "CS224u, Stanford, Spring 2019"

tf.enable_eager_execution()

@pytest.mark.parametrize("arg, expected", [
    [
        np.array([0.0, 0.25, 0.75]),
        np.array([0.22721977, 0.29175596, 0.48102426])
    ]
])
def test_softmax(arg, expected):
    result = utils.softmax(arg).round(8)
    expected = expected.round(8)
    assert np.array_equal(result, expected)

@pytest.mark.parametrize("arg, expected", [
    [-1, 0],
    [np.array([-1.0, 1.0]), np.array([0.0, 0.0])]
])
def test_d_tanh(arg, expected):
    assert np.array_equal(utils.d_tanh(arg), expected)

def test_randvec():
    x = utils.randvec(10)
    assert len(x) == 10

def test_randmatrix():
    X = utils.randmatrix(10, 20)
    assert X.shape == (10, 20)

def test_safe_macro_f1():
    y = [1, 1, 2, 2, 1]
    y_pred = [1, 2, 2, 1, 1]
    utils.safe_macro_f1(y, y_pred)

@pytest.mark.parametrize("arg, expected", [
    [
        np.array([[1.0, 0.0], [0.0, 1.0]]),
        np.array([[0.0, 0.0], [0.0, 0.0]])
    ]
])
def test_log_of_array_ignoring_zeros(arg, expected):
    result = utils.log_of_array_ignoring_zeros(arg)
    return np.array_equal(result, expected)

def test_glove2dict():
    src_filename = os.path.join("data", "glove.6B", "glove.6B.50d.txt")
    data = utils.glove2dict(src_filename)
    assert len(data) == 400000

@pytest.mark.parametrize("X, n_words, expected", [
    [
        [["a", "b", "c"], ["b", "c", "d"]],
        None,
        ["$UNK", "a", "b", "c", "d"]
    ],
    [
        [["a", "b", "c"], ["b", "c", "d"]],
        2,
        ["$UNK", "b", "c"]
    ],
    [
        [],
        2,
        ["$UNK"]
    ]
])
def test_get_vocab(X, n_words, expected):
    result = utils.get_vocab(X, n_words=n_words)
    assert result == expected

@pytest.mark.parametrize("set_value", [True, False])
def test_fix_random_seeds_system(set_value):
    utils.fix_random_seeds(seed=42, set_system=set_value)
    x = np.random.random()
    utils.fix_random_seeds(seed=42, set_system=set_value)
    y = np.random.random()
    assert (x == y) == set_value

@pytest.mark.parametrize("set_value", [True, False])
def test_fix_random_seeds_pytorch(set_value):
    utils.fix_random_seeds(seed=42, set_torch=set_value)
    x = torch.rand(1)
    utils.fix_random_seeds(seed=42, set_torch=set_value)
    y = torch.rand(1)
    assert (x == y) == set_value

@pytest.mark.parametrize("set_value", [True, False])
def test_fix_random_seeds_tensorflow(set_value):
    utils.fix_random_seeds(seed=42, set_tensorflow=set_value)
    x = tf.random.uniform([1]).numpy()
    utils.fix_random_seeds(seed=42, set_tensorflow=set_value)
    y = tf.random.uniform([1]).numpy()
    assert (x == y) == set_value


import torch
from torch.utils.data import Dataset
import pandas as pd
from sklearn.model_selection import train_test_split

class SUSYDataset(Dataset):
    def __init__(self, root, train=True, transform=None):
        super().__init__()

        self.train = train
        self.transform = transform

        features = [
            "class",
            "lepton 1 pT",
            "lepton 1 eta",
            "lepton 1 phi",
            "lepton 2 pT",
            "lepton 2 eta",
            "lepton 2 phi",
            "missing energy magnitude",
            "missing energy phi",
            "MET_rel",
            "axial MET",
            "M_R",
            "M_TR_2",
            "R",
            "MT2",
            "S_R",
            "M_Delta_R",
            "dPhi_r_b",
            "cos(theta_r1)",
        ]
        df = pd.read_csv(root + "/SUSY.csv", header=None)
        df.columns = features

        y = df["class"]
        X = df.drop("class", axis=1)

        # Perform stratified train/test split
        X_train, X_test, y_train, y_test = train_test_split(
            X, y, test_size=0.9, stratify=y
        )

        if train:
            X_train = (X_train - X_train.mean()) / X_train.std()
            self.data = X_train
            self.targets = y_train
        else:
            X_test = (X_test - X_test.mean()) / X_test.std()
            self.data = X_test
            self.targets = y_test

        self.data = torch.tensor(self.data.values, dtype=torch.float32)
        self.targets = torch.tensor(self.targets.values, dtype=torch.int64)

    def __getitem__(self, index):
        date, target = self.data[index], self.targets[index]

        if self.transform is not None:
            date = self.transform(date)

        return date, target

    def __len__(self):
        return len(self.data)


from updatebot.lib import util
from updatebot.cmdline import clientcfg

class AbstractUserInterface(object):
    """
    Abstract class to define user interface interface.
    """

    def ask(self, prompt, default=None):
        """
        Simple method for asking yes/no questions.
        @param prompt: string to prompt the user with.
        @type prompt: str
        @param default: Default value if no input is provided. This value will
                        be used if not running in interactive mode.
        @type default: boolean
        @return boolean value of the answer.
        @rtype boolean
        """

        raise NotImplementedError

class UserInterface(AbstractUserInterface):
    """
    Basic user interface class.
    @param cfg: client config object
    @type cfg: updatebot.cmdline.clientcfg.UpdateBotClientConfig
    """

    def __init__(self, cfg=None):
        if cfg is None:
            self.cfg = clientcfg.UpdateBotClientConfig()
        else:
            self.cfg = cfg

    def ask(self, prompt, default=None):
        """
        Simple method for asking yes/no questions.
        @param prompt: string to prompt the user with.
        @type prompt: str
        @param default: Default value if no input is provided. This value will
                        be used if not running in interactive mode.
        @type default: boolean
        @return boolean value of the answer.
        @rtype boolean
        """

        if not self.cfg.interactive:
            return default

        if default is True:
            prompt += ' [Y/n]:'
        elif default is False:
            prompt += ' [y/N]:'

        return util.askYn(prompt, default)


import argparse
import tensorflow as tf
import os

# Freeze graph. Replaces variables to constants. Removes training-only ops.
def freeze_graph(net, ckpt, tool, out_node, out_graph='frozen_graph.pb'):
    os.system('python ' + tool +
              ' --input_graph ' + net +
              ' --input_checkpoint ' + ckpt +
              ' --output_graph ' + out_graph +
              ' --output_node_names ' + out_node)

def optimize_for_inference(dtype, tool, in_node, out_node, out_graph='optimized_graph.pb',
                           frozen_graph='frozen_graph.pb'):
    os.system('python ' + tool +
              ' --input ' + frozen_graph +
              ' --output ' + out_graph +
              ' --frozen_graph True ' +
              ' --input_names ' + in_node +
              ' --output_names ' + out_node +
              ' --placeholder_type_enum ' + str(dtype.as_datatype_enum))

def fuse_constants(tool, in_graph, out_graph, in_node, out_node):
    os.system(tool + ' --in_graph=' + out_graph + \
                     ' --out_graph=' + out_graph + \
                     ' --inputs=' + in_node + \
                     ' --outputs=' + out_node + \
                     ' --transforms="fold_constants(ignore_errors=True) sort_by_execution_order"')

def prepare_for_dnn(net, ckpt, freeze_graph_tool, optimizer_tool, transform_graph_tool,
                    input_node_name, output_node_name, out_graph, dtype):
    freeze_graph(net, ckpt, freeze_graph_tool, output_node_name)
    optimize_for_inference(dtype, optimizer_tool, input_node_name, output_node_name, out_graph)
    fuse_constants(transform_graph_tool, out_graph, out_graph, input_node_name, output_node_name)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Script for preparing serialized '
                                                 'TensorFlow graph to import into DNN. '
                                                 'Modified graph still may be used in TensorFlow.')
    parser.add_argument('-frz', dest='freeze_graph_tool', required=True,
                        help='Path to freeze_graph.py tool')
    parser.add_argument('-opt', dest='optimizer_tool', required=True,
                        help='Path to optimize_for_inference.py tool')
    parser.add_argument('-tr', dest='transform_graph_tool', required=True,
                        help='Path to transform_graph tool')
    parser.add_argument('-net', dest='net', required=True,
                        help='Path serialized graph by tf.train.write_graph()')
    parser.add_argument('-ckpt', dest='ckpt', required=True,
                        help='Path saved checkpoint by saver.save()')
    parser.add_argument('-in_node', dest='input_name', required=True,
                        help='Input op name')
    parser.add_argument('-out_node', dest='output_name', required=True,
                        help='Output op name')
    parser.add_argument('-o', dest='output', required=True,
                        help='Output graph name')
    args = parser.parse_args()

    prepare_for_dnn(args.net, args.ckpt, args.freeze_graph_tool,
                    args.optimizer_tool, args.transform_graph_tool,
                    args.input_name, args.output_name, args.output)


import os
import logging
from logging.handlers import SMTPHandler, RotatingFileHandler
import pprint

from elasticsearch import Elasticsearch
from flask import Flask, session, request, current_app, redirect
from flask_sqlalchemy import SQLAlchemy
from flask_migrate import Migrate
from flask_login import LoginManager, current_user
from flask_mail import Mail
from flask_moment import Moment
from flask_babel import Babel
from flask_admin import Admin, BaseView, expose
from flask_admin.contrib.sqla import ModelView

from config import Config
from app.search import get_mappings, insert_mapping, delete_mapping

# Turn off autoflush to let review editing to be saved in session.dirty
db = SQLAlchemy(session_options={"autoflush": False})
migrate = Migrate()
login = LoginManager()
login.login_view = 'auth.login'
mail = Mail()
moment = Moment()
babel = Babel()
admin = Admin()

def create_app(config_class=Config):
    app = Flask(__name__)
    app.config.from_object(config_class)

    db.init_app(app)
    migrate.init_app(app, db)
    login.init_app(app)
    mail.init_app(app)
    moment.init_app(app)
    babel.init_app(app)
    admin.init_app(app)
    app.elasticsearch = Elasticsearch([app.config['ELASTICSEARCH_URL']]) if app.config['ELASTICSEARCH_URL'] else None

    from app.errors.handlers import bp as errors_bp
    app.register_blueprint(errors_bp)

    from app.auth import bp as auth_bp
    app.register_blueprint(auth_bp, url_prefix='/auth')

    from app.main import bp as main_bp
    app.register_blueprint(main_bp)

    if not app.debug and not app.testing:
        if app.config['MAIL_SERVER']:
            auth = None
            if app.config['MAIL_USERNAME'] or app.config['MAIL_PASSWORD']:
                auth = (app.config['MAIL_USERNAME'], app.config['MAIL_PASSWORD'])
            secure = None
            if app.config['MAIL_USE_TLS']:
                secure = ()
            mail_handler = SMTPHandler(
                mailhost=(app.config['MAIL_SERVER'], app.config['MAIL_PORT']),
                fromaddr='no-reply@' + app.config['MAIL_SERVER'],
                toaddrs=app.config['ADMINS'], subject='Whisky Blog failure',
                credentials=auth, secure=secure)
            mail_handler.setLevel(logging.ERROR)
            app.logger.addHandler(mail_handler)
        if app.config['LOG_TO_STDOUT']:
            stream_handler = logging.StreamHandler()
            stream_handler.setLevel(logging.INFO)
            app.logger.addHandler(stream_handler)
        else:
            if not os.path.exists('logs'):
                os.mkdir('logs')
            file_handler = RotatingFileHandler('logs/WhiskyBlog.logs', maxBytes=10240, backupCount=10)
            file_handler.setFormatter(logging.Formatter(
                '%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]'))
            file_handler.setLevel(logging.INFO)
            app.logger.addHandler(file_handler)

        app.logger.setLevel(logging.INFO)
        app.logger.info('Whisky Blog startup')

    return app

# Looks for the language the current flask app is set to.
@babel.localeselector
def get_locale():
    try:
        language = session['language']
    except KeyError:
        language = None
    if language is not None:
        return language
    return request.accept_languages.best_match(current_app.config['LANGUAGES'])

from app import models

"""Create custom admin views for `User`, `Review` and `Tag` models"""

class BaseModelView(ModelView):
    def is_accessible(self):
        return current_user.is_authenticated and current_user.id == 1

class UserView(BaseModelView):
    column_exclude_list = ['password_hash']
    can_create = False
    can_edit = True

class ReviewView(BaseModelView):
    can_create = False
    can_edit = False

class TagView(BaseModelView):
    @expose('/bulk/')
    def bulk_view(self):
        from app.main.info import all_tags
        for t in all_tags:
            tag = self.model.query.filter_by(name=t[0]).first()
            if not tag:
                self.session.add(self.model(name=t[0]))
        self.session.commit()
        return redirect('/admin/tag')

class SearchView(BaseView):
    @expose('/')
    def index(self):
        maps = get_mappings()
        return self.render('admin/search.html', mapping=pprint.pformat(maps))

    @expose('/insert/')
    def insert(self):
        # insert elasticsearch mapping
        insert_mapping('review')
        return redirect('/admin/search')

    @expose('/delete/')
    def delete(self):
        # delete elasticsearch mapping
        delete_mapping('review')
        return redirect('/admin/search')

    def is_accessible(self):
        return current_user.is_authenticated and current_user.id == 1

admin.add_view(UserView(models.User, db.session))
admin.add_view(ReviewView(models.Review, db.session))
admin.add_view(TagView(models.Tag, db.session))
admin.add_view(SearchView(name='Search', endpoint='search'))


#! /user/bin/env python
# encoding:utf-8
import requests
import pandas as pd
import json

markets = ['szmb','szsme','szcn','shmb']
market_names = ['深市主板','中小板','创业板','沪市']
market_name_dict = dict(zip(markets, market_names))

def fetch_prbookinfo_from_cninfo(market):
    #从巨潮资讯获取报告披露时间信息
    #market是市场代码
    print('开始获取'+market_name_dict[market]+'数据.......')
    base_url = 'http://www.cninfo.com.cn/cninfo-new/information/prbookinfo-1'
    data = {'sectionTime': '2016-12-31', 'market': market, 'orderClos': '', 'isDesc': '', 'stockCode': '',
            'firstTime': u'请输入预约披露时间', 'lastTime': '请输入实际披露时间'}
    r = requests.post(base_url,data=data)
    if r.status_code != 200:
        raise Exception('Error in get data,status_code is:'+str(r.status_code))
    datas = json.loads(r.text)
    df = pd.DataFrame(datas)
    print('共获取'+market_name_dict[market]+'股票 '+ str(len(df)) + '只')
    return df

def fetch_no_annualReport_stocks():
    # 筛选没有出年报的股票
    df = None
    for market in markets:
        t_df = fetch_prbookinfo_from_cninfo(market)
        #实际披露时间形如2017-03-17，至少有10个字符
        t_df = t_df[t_df.f006d_0102.str.len() < 10]
        print(market_name_dict[market]+'尚未披露年报的股票有' + str(len(t_df)) + '只')
        if df is None:
            df = t_df
        else:
            df = df.append(t_df,ignore_index=True)
    print('还没有披露年报的股票一共有 '+ str(len(df))+ '只')
    return df

def main():
    hdf = fetch_no_annualReport_stocks()
    print(hdf)
    # filename = u'F:/firecapital/数据/output20170501/普通总列表.xlsx'
    # df = pd.read_excel(filename,converters = {'code':str})
    # df = df[['period','code','weight']]
    # if len(df.code[0]) > 6:
    #     df.code = df.code.str.slice(2,8)

if __name__ == '__main__':
    main()


"""
This is an attempt to send some dummy sensor readings.
"""
import random
try:
    import asyncio
except ImportError:
    import uasyncio as asyncio  # noqa
from mavlite import MavLink, UART

from time import monotonic

try:
    import board  # noqa
    _uart = UART(tx=board.TX, rx=board.RX, baudrate=230400)
except ImportError:
    _uart = UART(tx=17, rx=16, baudrate=230400)

ml = MavLink(
    message_ids=[  # Message includes.
        132,  # DISTANCE_SENSOR
    ],
)

async def distance_sensor():
    """
    This will send the distance sensor messages to the flight controller.
    """
    while True:  # Transmit loop
        try:
            await ml.send_message(
                message_id=132,  # DISTANCE_SENSOR
                payload=[
                    int(monotonic()),  # boot_time
                    1,  # min_distance
                    100,  # max_distance
                    random.randint(1, 100),  # current_distance
                    0,  # type
                    0,  # id
                    25,  # orientation  # Make fucking sure this matched the value in mission planner...
                    255,  # covariance
                    20,  # horizontal_fov
                    20,  # vertical_fov
                    0,  # quaternion
                    0,  # signal_quality
                ],
                debug=True
            )
        except KeyboardInterrupt:
            break

async def main(uart_):
    """
    Test mainloop.
    """

    tasks = await ml.io_buffers(uart_, debug=True)  # [write_loop, read_loop, heartbeat_loop, command_listener]
    await asyncio.gather(distance_sensor(), *tasks[0:3])

asyncio.run(main(_uart))


import sys
import schema as sch
import mi_main as mim

test_status = 0

def check_decl(decls, decl):
    global test_status
    if decl in decls:
        print(decl + " present   OK")
    else:
        print(decl + " missing   FAIL")
        test_status = 1

def check_type(obj, t):
    global test_status
    if type(obj) == t:
        print(str(type(obj)) + " found   OK")
    else:
        print(str(type(obj)) + " found, wrong type, should be" + str(t) + "   FAIL")
        test_status = 1

def get_last_word(line):
    words = line.split()
    member = words[len(words)-1]
    #remove ';' from the end, also removes trailing white space
    member = member.split(";")[0]
    #remove '[]' in case of arrays
    member = member.split("[")[0]
    member = member.split("]")[0]
    #remove '()' in case of function
    member = member.split("(")[0]
    member = member.split(")")[0]
    #remove ',' in case of function parameter
    member = member.split(",")[0]

    return member

#does naive parsing to gather class names and member names
def get_class_names(mof_file):
    classes = []
    with open(mof_file) as f:
        file_lines = f.readlines()

    inside_class = False
    inside_func = False
    class_def = []
    class_mem = []
    func_mem = []
    func_name = ""
    for line in file_lines:
        if line.find("class") != -1:
            class_def.append(line.split()[1])

        if line.find("{") != -1:
            inside_class = True
        if line.find("}") != -1:
            inside_class = False
            class_def.append(class_mem)
            classes.append(class_def)
            class_def = []
            class_mem = []

        if line.find("(") != -1:
            if line.find(")") == -1:
                inside_func = True
                func_name = get_last_word(line)

        if line.find(")") != -1:
            if line.find("(") == -1:
                inside_func = False
                class_mem.append([func_name, func_mem])
                func_mem = []
                func_name = ""

        if inside_class:
            if inside_func:
                last_word = get_last_word(line)
                if func_name != last_word:
                    func_mem.append(last_word)
            else:
                if line.find(";") != -1:
                    #is it a function with no parameters
                    if line.find("()") != -1:
                        class_mem.append([get_last_word(line), []])
                    else:
                        mem_name = get_last_word(line)
                        if(mem_name != ""):
                            class_mem.append(mem_name)

    return classes

mof_file = sys.argv[1]
print("Testing mof file ..." + mof_file)
all_decl = dir(sch)
classes = get_class_names(mof_file)

#check if declarations are correct
check_decl(all_decl, "Weak_qual_decl")
check_type(sch.Weak_qual_decl, sch.omi.MI_QualifierDecl)

check_decl(all_decl, "Weak_qual_decl_value")
check_type(sch.Weak_qual_decl_value, sch.omi.MI_Boolean)

check_decl(all_decl, "Write_qual_decl")
check_type(sch.Weak_qual_decl, sch.omi.MI_QualifierDecl)

check_decl(all_decl, "Write_qual_decl_value")
check_type(sch.Weak_qual_decl_value, sch.omi.MI_Boolean)

#check if class declarations are present
for clazz in classes:
    clazz_name = clazz[0]
    clazz_memb = clazz[1]

    check_decl(all_decl, clazz_name + "_quals")
    check_decl(all_decl, clazz_name + "_properties")
    check_decl(all_decl, clazz_name + "_methods")
    check_decl(all_decl, clazz_name + "_functions")
    check_decl(all_decl, clazz_name + "_class")

    for memb in clazz_memb:
        if type(memb) == list:
            check_decl(all_decl, clazz_name + "_" + memb[0] + "_quals")
            check_decl(all_decl, clazz_name + "_" + memb[0] + "_params")
            check_decl(all_decl, clazz_name + "_" + memb[0] + "_method")

            for param in memb[1]:
                check_decl(all_decl, clazz_name + "_" + memb[0] + "_" + param + "_quals")
                check_decl(all_decl, clazz_name + "_" + memb[0] + "_" + param + "_param")
        else:
            check_decl(all_decl, clazz_name + "_" + memb + "_quals")
            check_decl(all_decl, clazz_name + "_" + memb + "_prop")

exit(test_status)


import csv
from io import StringIO
from zipfile import ZipFile
from chapter3.mongo_cache import MongoCache

class AlexaCallback():

    def __init__(self, max_urls=1000):
        self.max_urls = max_urls
        self.seed_url = 'http://s3.amazonaws.com/alexa-static/top-1m.csv.zip'
        self.csv_file = '/home/vagrant/Downloads/top-1m.csv'

    def __call__(self):

        urls = []
        # cache = MongoCache()
        with open(self.csv_file, 'r') as zf:
            for _, website in csv.reader(zf):
                # if 'http://' + website not in cache:
                urls.append('http://' + website)
                if len(urls) == self.max_urls:
                    break

        return urls

if __name__ == '__main__':
    result = AlexaCallback()()
    print(result)


strategyGuide = open('input.tXt')

rounds = strategyGuide.readlines()

index = 0

size = len(rounds)

totalScore = 0
currentScore = 0

while(index < size):
    #print(rounds[indeX])
    myPlay = rounds[index][2]
    oppPlay = rounds[index][0]

    #print(myPlay)
    #print(oppPlay)

    if(myPlay == 'X'): #rock
        #print("rock +1")
        currentScore += 1
        if(oppPlay == 'C'): #scissors
            #print("rock YeXts scissors. +6")
            currentScore += 6
        elif(oppPlay == 'A'): #rock
            #print("Tie. +3")
            currentScore +=3
    elif(myPlay == 'Y'): #pXper
        #print("PXper +2")
        currentScore += 2
        if(oppPlay == 'A'): #rock
            #print("PXper YeXts rock. +6")
            currentScore += 6
        elif(oppPlay == 'B'): #pXper
            currentScore +=3
    elif(myPlay == 'Z'): #scissors
        #print("scissors +3")
        currentScore += 3
        if(oppPlay == 'B'): #pXper
            #print("scissors YeXts poZk. +6")
            currentScore += 6
        elif(oppPlay == 'C'): #scissors
            currentScore +=3

    #print("SZore for round: " + str(currentScore))
    totalScore += currentScore
    #print(totalScore)
    currentScore = 0
    index += 1
    #print()

print(totalScore)


import math,sys,uuid
import numpy as np
import numpy.matlib
from multiprocessing import Pool
from multiprocessing import shared_memory
from calibration_tools import *

def globalize(func):
  def result(*args, **kwargs):
    return func(*args, **kwargs)
  result.__name__ = result.__qualname__ = uuid.uuid4().hex
  setattr(sys.modules[result.__module__], result.__name__, result)
  return result

def analysis_uvwdir_loop(skymodel,clusterfile,uvwfile,rhofile,solutionsfile,z_solfile,flow=110,fhigh=170,ra0=0,dec0=math.pi/2,tslots=10,alpha=0.1,Nparallel=4):
    # ra0,dec0: phase center (rad)
    # tslots: -t option
    # alpha: spatial constraint regularization parameter
    # Nparallel=number of parallel jobs to use
    flow=flow*1e6
    fhigh=fhigh*1e6

    # if 1, IQUV, else only I
    fullpol=0
    loop_in_r=False # use 8 blocks instead of looping

    # read Z (global) solutions to get metadata for consensus polynomial
    N,f0,Ne,K1,Z1=read_global_solutions(z_solfile)
    # N: stations
    # Ne: consensus poly terms, same as -P parameter in sagecal
    # baselines
    B=int(N*(N-1)/2)
    # reference freq (for consensus polynomial)
    assert(f0>=flow and f0<=fhigh) # mean of all freqs
    #%%%%%%%%%%%%%%%%%% consensus polynomial info
    Nf=8 # no. of freqs: make sure to match all data
    f=np.linspace(flow,fhigh,Nf)
    polytype=1 # 0: ordinary, 1: Bernstein

    # read solutions file (also get the frequency(MHz)) J: Kx2N Nt x 2 (2Nx2 blocks Nt times)
    freq,J=readsolutions(solutionsfile)
    # read sky model Ct: Kx T x 4 (each row XX,XY,YX,YY)
    K,Ct=skytocoherencies(skymodel,clusterfile,uvwfile,N,freq,ra0,dec0)
    assert(K==K1)

    # ADMM rho, per each direction, scale later
    # scale rho linearly with sI
    rho=read_rho(rhofile,K)

    # read u,v,w,xx(re,im), xy(re,im) yx(re,im) yy(re,im)
    XX,XY,YX,YY=readuvw(uvwfile)
    # create shared memory equal to XX,XY,YX,YY buffers for parallel processing
    shmXX=shared_memory.SharedMemory(create=True,size=XX.nbytes)
    shmXY=shared_memory.SharedMemory(create=True,size=XY.nbytes)
    shmYX=shared_memory.SharedMemory(create=True,size=YX.nbytes)
    shmYY=shared_memory.SharedMemory(create=True,size=YY.nbytes)
    # create arrays that can be used in multiprocessing
    XX0=np.ndarray(XX.shape,dtype=XX.dtype,buffer=shmXX.buf)
    XY0=np.ndarray(XY.shape,dtype=XY.dtype,buffer=shmXY.buf)
    YX0=np.ndarray(YX.shape,dtype=YX.dtype,buffer=shmYX.buf)
    YY0=np.ndarray(YY.shape,dtype=YY.dtype,buffer=shmYY.buf)
    # how many timeslots to use per calibration (-t option)
    T=tslots
    Ts=int(XX.shape[0]//(B*T))

    # check this agrees with solutions
    nx,ny=J[0].shape
    if nx<2*N*Ts:
     print('Error: solutions size does not match with data size')
     exit

    # which frequency index to work with
    fidx=np.argmin(np.abs(f-freq))

    # addition to Hessian
    Hadd=np.zeros((K,4*N,4*N),dtype=np.float32)
    for ci in range(K):
     # note: F is dependent on rho when alpha!=0
     # example: making F=rand(2N,2N) makes performance worse
     F,P=consensus_poly(Ne,N,f,f0,fidx,polytype=polytype,rho=rho[ci],alpha=alpha)
     FF=np.matmul(F.transpose(),F)
     if alpha>0.0:
       PP=np.matmul(P.transpose(),P)
       H11=0.5*rho[ci]*FF+0.5*alpha*rho[ci]*rho[ci]*PP
       H12=0.5*FF+0.5*alpha*rho[ci]*PP
       H21=H12
       H22=-0.5/rho[ci]*(np.eye(2*N)-FF)+0.5*alpha*PP
       Htilde=H11-np.matmul(H12,np.matmul(np.linalg.pinv(H22),H21))
       Hadd[ci]=np.kron(np.eye(2),Htilde)
     else:
       Hadd[ci]=0.5*rho[ci]*np.kron(np.eye(2),np.matmul(FF,np.eye(2*N)+np.matmul(np.linalg.pinv(np.eye(2*N)-FF),FF)))

############################# loop over timeslots
############################# local function
    @globalize
    def process_chunk(ncal):
        ts=ncal*T
        print('%d %d %d'%(ts,Ts,ncal))
        R=np.zeros((2*B*T,2),dtype=np.csingle)
        R[0:2*B*T:2,0]=XX[ts*B:ts*B+B*T]
        R[0:2*B*T:2,1]=XY[ts*B:ts*B+B*T]
        R[1:2*B*T:2,0]=YX[ts*B:ts*B+B*T]
        R[1:2*B*T:2,1]=YY[ts*B:ts*B+B*T]

        # D_Jgrad K x 4Nx4N tensor
        H=Hessianres(R,Ct[:,ts*B:ts*B+B*T],J[:,ncal*2*N:ncal*2*N+2*N],N)
        H+=Hadd

        # set to zero
        XX0[ts*B:ts*B+B*T]=0
        XY0[ts*B:ts*B+B*T]=0
        YX0[ts*B:ts*B+B*T]=0
        YY0[ts*B:ts*B+B*T]=0

        if loop_in_r:
          for r in range(8):
            # dJ: K x 4NxB tensor
            dJ=Dsolutions(Ct[:,ts*B:ts*B+B*T],J[:,ncal*2*N:ncal*2*N+2*N],N,H,r)
            # dR: 4B x B (sum up all K)
            dR=Dresiduals(Ct[:,ts*B:ts*B+B*T],J[:,ncal*2*N:ncal*2*N+2*N],N,dJ,0,r) # 0 for not adding I to dR
            # find mean value over columns
            dR11=np.mean(dR[0:4*B:4],axis=0)
            dR11=np.squeeze(np.matlib.repmat(dR11,1,T))
            XX0[ts*B:ts*B+B*T] +=dR11
            dR11=np.mean(dR[3:4*B:4],axis=0)
            dR11=np.squeeze(np.matlib.repmat(dR11,1,T))
            YY0[ts*B:ts*B+B*T] +=dR11
            if fullpol:
              dR11=np.mean(dR[1:4*B:4],axis=0)
              dR11=np.squeeze(np.matlib.repmat(dR11,1,T))
              XY0[ts*B:ts*B+B*T] +=dR11
              dR11=np.mean(dR[2:4*B:4],axis=0)
              dR11=np.squeeze(np.matlib.repmat(dR11,1,T))
              YX0[ts*B:ts*B+B*T] +=dR11
        else:
          # dJ: 8 x K x 4NxB tensor
          dJ=Dsolutions_r(Ct[:,ts*B:ts*B+B*T],J[:,ncal*2*N:ncal*2*N+2*N],N,H)
          # dR: 8 x 4B x B (sum up all K)
          dR=Dresiduals_r(Ct[:,ts*B:ts*B+B*T],J[:,ncal*2*N:ncal*2*N+2*N],N,dJ,0) # 0 for not adding I to dR
          # find mean value over columns
          for r in range(8):
            dR11=np.mean(dR[r,0:4*B:4],axis=0)
            dR11=np.squeeze(np.matlib.repmat(dR11,1,T))
            XX0[ts*B:ts*B+B*T] +=dR11
            dR11=np.mean(dR[r,3:4*B:4],axis=0)
            dR11=np.squeeze(np.matlib.repmat(dR11,1,T))
            YY0[ts*B:ts*B+B*T] +=dR11
            if fullpol:
              dR11=np.mean(dR[r,1:4*B:4],axis=0)
              dR11=np.squeeze(np.matlib.repmat(dR11,1,T))
              XY0[ts*B:ts*B+B*T] +=dR11
              dR11=np.mean(dR[r,2:4*B:4],axis=0)
              dR11=np.squeeze(np.matlib.repmat(dR11,1,T))
              YX0[ts*B:ts*B+B*T] +=dR11
############################# end local function
############################# loop over timeslots

    # create pool
    pool=Pool(Nparallel)
    pool.map(process_chunk,range(Ts))
    pool.close()
    pool.join()

    # copy back from shared memory
    XX[:]=XX0[:]
    XY[:]=XY0[:]
    YX[:]=YX0[:]
    YY[:]=YY0[:]
    # release shared memory
    shmXX.close()
    shmXX.unlink()
    shmXY.close()
    shmXY.unlink()
    shmYX.close()
    shmYX.unlink()
    shmYY.close()
    shmYY.unlink()

    scalefactor=8*(N*(N-1)/2)*T
    # scale by 8*(N*(N-1)/2)*T
    writeuvw('fff',scalefactor*XX,scalefactor*XY,scalefactor*YX,scalefactor*YY)

if __name__ == '__main__':
  # args skymodel clusterfile uvwfile rhofile solutionsfile z_solutions_file freq_low(MHz) freq_high(MHz) ra0 dec0 tslots alpha parallel_jobs
  import sys
  argc=len(sys.argv)
  if argc==13:
   analysis_uvwdir_loop(sys.argv[1],sys.argv[2],sys.argv[3],sys.argv[4],sys.argv[5],sys.argv[6],float(sys.argv[7]),float(sys.argv[8]),float(sys.argv[9]),float(sys.argv[10]),int(sys.argv[11]),float(sys.argv[12]))
  elif argc==14:
   analysis_uvwdir_loop(sys.argv[1],sys.argv[2],sys.argv[3],sys.argv[4],sys.argv[5],sys.argv[6],float(sys.argv[7]),float(sys.argv[8]),float(sys.argv[9]),float(sys.argv[10]),int(sys.argv[11]),float(sys.argv[12]),int(sys.argv[13]))
  else:
   print("Usage: python %s skymodel clusterfile uvwfile rhofile solutionsfile z_solutions_file freq_low freq_high ra0 dec0 tslots alpha parallel_jobs"%(sys.argv[0]))
  exit()


from typing import List, Optional, Sequence

from llama_index.llms.types import ChatMessage, MessageRole

BOS, EOS = "<s>", "</s>"
B_INST, E_INST = "[INST]", "[/INST]"
B_SYS, E_SYS = "<<SYS>>\n", "\n<</SYS>>\n\n"
DEFAULT_SYSTEM_PROMPT = """\
You are a helpful, respectful and honest assistant. \
Always answer as helpfully as possible and follow ALL given instructions. \
Do not speculate or make up information. \
Do not reference any given instructions or context. \
"""

def messages_to_prompt(
    messages: Sequence[ChatMessage], system_prompt: Optional[str] = None
) -> str:
    string_messages: List[str] = []
    if messages[0].role == MessageRole.SYSTEM:
        # pull out the system message (if it exists in messages)
        system_message_str = messages[0].content or ""
        messages = messages[1:]
    else:
        system_message_str = system_prompt or DEFAULT_SYSTEM_PROMPT

    system_message_str = f"{B_SYS} {system_message_str.strip()} {E_SYS}"

    for i in range(0, len(messages), 2):
        # first message should always be a user
        user_message = messages[i]
        assert user_message.role == MessageRole.USER

        if i == 0:
            # make sure system prompt is included at the start
            str_message = f"{BOS} {B_INST} {system_message_str} "
        else:
            # end previous user-assistant interaction
            string_messages[-1] += f" {EOS}"
            # no need to include system prompt
            str_message = f"{BOS} {B_INST} "

        # include user message content
        str_message += f"{user_message.content} {E_INST}"

        if len(messages) > (i + 1):
            # if assistant message exists, add to str_message
            assistant_message = messages[i + 1]
            assert assistant_message.role == MessageRole.ASSISTANT
            str_message += f" {assistant_message.content}"

        string_messages.append(str_message)

    return "".join(string_messages)

def completion_to_prompt(completion: str, system_prompt: Optional[str] = None) -> str:
    system_prompt_str = system_prompt or DEFAULT_SYSTEM_PROMPT

    return (
        f"{BOS} {B_INST} {B_SYS} {system_prompt_str.strip()} {E_SYS} "
        f"{completion.strip()} {E_INST}"
    )


import json
import ulab as np
import gc

gc.enable()
gc.threshold(30000)

__version__ = '1.0.2'

class Model():
    def __init__(self, path):
        f = open(path)
        data = f.read()
        self.model_dict = json.loads(data)
        del data
        for i in range(len(self.model_dict['layers'])):
            if self.model_dict['layers'][i]['class_name'] == 'Dense':
                self.model_dict['layers'][i]['weights'][0]=np.array(self.model_dict['layers'][i]['weights'][0])
                if type(self.model_dict['layers'][i]['weights'][1])==list:
                    self.model_dict['layers'][i]['weights'][1]=np.array(self.model_dict['layers'][i]['weights'][1])
            elif self.model_dict['layers'][i]['class_name'] == 'Conv1D':
                self.model_dict['layers'][i]['weights'][0]=np.array(self.model_dict['layers'][i]['weights'][0])
                if type(self.model_dict['layers'][i]['weights'][1])==list:
                    self.model_dict['layers'][i]['weights'][1]=np.array(self.model_dict['layers'][i]['weights'][1])

    def predict(self, x):
        for layer in self.model_dict['layers']:
            if layer['class_name'] == 'Dense':
                x = np.dot(x,layer['weights'][0])
                x = x + layer['weights'][1]
                x = activation(x,f=layer['activation'])
            elif layer['class_name'] == 'Reshape':
                x = reshape(x,tuple(layer['target_shape']))
            elif layer['class_name'] == 'Conv1D':
                x = conv1D(x,layer['weights'][0],
                               layer['weights'][1],
                               kernel_size=layer['kernel_size'],
                               strides=layer['strides'],
                               padding=layer['padding'])
                x = activation(x,f=layer['activation'])
            elif layer['class_name'] == 'MaxPooling1D':
                x = maxPooling1D(x,pool_size=layer['pool_size'],strides=layer['strides'])
            elif layer['class_name'] == 'AveragePooling1D':
                x = averagePooling1D(x,pool_size=layer['pool_size'],strides=layer['strides'])
            elif layer['class_name'] == 'GlobalMaxPooling1D':
                x = globalMaxPooling1D(x)
            elif layer['class_name'] == 'GlobalAveragePooling1D':
                x = globalAveragPooling1D(x)
            elif layer['class_name'] == 'Flatten':
                x = flatten(x)
        return x

    def predict_classes(self, x):
        activation = self.model_dict['layers'][-1].get('activation')
        output = self.predict(x)
        if activation=='softmax':
            output = np.argmax(output,axis=1)
        elif activation=='sigmoid':
            output = np.array(output>=0.5,dtype=np.uint16)
        return output

def activation(x,f='relu'):
    if type(x)==list:
        for i in range(len(x)):
            np.activation(x[i],f=f)
    else:
        np.activation(x,f=f)
    return x

def conv1D(x,w,b,kernel_size=3,strides=1,padding='valid'):
    if type(x)==list:
        output = []
        for i in range(len(x)):
            output.append(np.conv1D(x[i],w,b,kernel_size=kernel_size,strides=strides,padding=padding))
        return output
    else:
        raise ValueError('First input should be list')

def maxPooling1D(x,pool_size=2,strides=0):
    if type(x)==list:
        output = []
        for i in range(len(x)):
            output.append(np.maxPooling1D(x[i],pool_size=pool_size,strides=strides))
        return output
    else:
        raise ValueError('First input should be list')

def averagePooling1D(x,pool_size=2,strides=0):
    if type(x)==list:
        output = []
        for i in range(len(x)):
            output.append(np.averagePooling1D(x[i],pool_size=pool_size,strides=strides))
        return output
    else:
        raise ValueError('First input should be list')

def globalMaxPooling1D(x):
    if type(x)==list:
        output = []
        for i in range(len(x)):
            output.append(np.globalMaxPooling1D(x[i]))
        output = np.array(output)
        return output
    else:
        raise ValueError('Input should be list')

def globalAveragePooling1D(x):
    if type(x)==list:
        output = []
        for i in range(len(x)):
            output.append(np.globalAveragePooling1D(x[i]))
        output = np.array(output)
        return output
    else:
        raise ValueError('Input should be list')

def flatten(x):
    if type(x)==list:
        output = []
        for i in range(len(x)):
            output.append(x[i].flatten())
        output = np.array(output)
        return output
    else:
        raise ValueError('First input should be list')

def reshape(x,shape):
    if type(x)==type(np.array([])):
        if x.shape()[0]==1:
            output = [np.array(x).reshape(shape)]
        else:
            output = []
            for i in x:
                output.append(i.reshape(shape))
        return output
    else:
        raise ValueError('Input should be ndarray')


import asyncio
import bz2
import contextlib
import logging

import player.codec
import player.terminal
import player.tools

class Session:
	'''
	Session represent one client
	also separate sessions can replay different files
	'''

	def __init__(self, session_id: str, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, filename: str):
		self._logger = logging.getLogger('{}[{}]'.format(self.__class__.__name__, session_id))
		self.session_id = session_id
		self.filename = filename
		self._writer = writer
		self._reader = reader

		self._run_future = None # type: asyncio.Future

	async def run(self) -> None:
		'''
		Wrapper which allow us to terminate session object
		'''
		self._run_future = asyncio.ensure_future(self._run())
		await self._run_future

	async def _run(self) -> None:
		'''
		Run loop which will stream frame by frame to writer
		'''
		self._logger.debug('Reading file = %s', self.filename)
		with bz2.BZ2File(self.filename, 'rb') as file:
			await self._clear_screen()

			metadata = player.codec.get_file_metadata(file)
			sleep_time = 1.0 / metadata.frame_rate

			stop_watch = player.tools.StopWatch()
			for frame in player.codec.get_frames(file, metadata):
				# Move cursor to top left corner
				self._writer.write(player.terminal.RESET_CURSOR)
				self._writer.write(frame)
				await self._writer.drain()

				# We will have to correct sleep time and
				# decrease how long it take to load one frame from it
				t = max(0, stop_watch.lap() - sleep_time)
				if sleep_time < t:
					self._logger.error('Sleep time is less then reading overhead by = %.2f', t)
				await asyncio.sleep(max(0, sleep_time - t))

			await self._clear_screen()

	async def terminate(self) -> None:
		'''
		Close writer and clenaup after session
		'''
		self._logger.debug('Terminating of session requested.')
		if self._run_future is not None and not self._run_future.done():
			self._run_future.cancel()

			with contextlib.suppress(asyncio.CancelledError):
				await self._run_future

		self._writer.close()

	async def _clear_screen(self) -> None:
		'''
		Write clearing of screen
		'''
		# First white all empty lines
		self._writer.write(player.terminal.CLEAR)
		# And then reset cursor to upper left
		self._writer.write(player.terminal.RESET_CURSOR)
		await self._writer.drain()


import copy
import json
import os.path
import torch
from transformers import BartTokenizer
from typing import List, Dict
from fire import Fire
from transformers import AutoTokenizer, PreTrainedTokenizer

def fix_random(seed):
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False
    torch.backends.cudnn.enabled = False

def load_json(file_path):
    data = []
    with open(file_path, 'r', encoding='utf-8') as f:
        for line in f.readlines():
            data.append(json.loads(line.strip()))
    return data

def write_json(file_path, data: List[Dict]):
    with open(file_path, 'w') as f:
        for line in data:
            f.write(json.dumps(line))
            f.write('\n')

def norm_strategy(strategy):
    norm_str = "-".join(strategy.split())
    return "@["+norm_str+"]"

def get_strategy(file_path, norm=False):
    with open(file_path,'r', encoding='utf-8') as f:
        data = json.load(f)
    data = [d.replace('[','').replace(']','') for d in data]
    if norm:
        data = [norm_strategy(d) for d in data]
    print('strategy: ', data)

    return data

def _norm(x):
    return ' '.join(x.strip().split()).lower()

#todo: add <helper> and <seeker> tokens to differentiate between the two (sometimes one speaker talks multiple times in the dataset)
def construct_conversational_dataset(
        file_path: str,
        tokenizer: BartTokenizer,
        add_cause: bool = False,
        with_strategy: bool = False,
        joint_strategy_utt: bool = True,
        load_from_cache: bool = True,
) -> str:

    split = 'train'
    if 'valid' in file_path:
        split = 'valid'
    elif 'test' in file_path:
        split = 'test'

    cached_preprocessed_file = file_path.replace(split, f"preprocessed_{split}")

    if os.path.exists(cached_preprocessed_file) and load_from_cache:
        print(f"loading preprocessed {split} dataset from {cached_preprocessed_file}")
        return cached_preprocessed_file

    data = load_json(file_path)
    print(f"parsing {file_path} file")
    sep_token = tokenizer.sep_token if tokenizer.sep_token is not None else " "
    print(f"setting {sep_token} as sep token")

    total_data = []
    is_train = 'train' in file_path

    # todo: check if required later:
    # if 'test' in file_path and self.model_type > 4:
    #     gt_strategy = read_pk('./final_data/test_extend_label.pk')
    #     if lookahead is not True:
    #         print("do not use lookahead! ")
    #         predict_strategy = read_pk('./final_data/wo_lookahead_predicted.pk')
    #     else:
    #         print('use lookahead! ')
    #         predict_strategy = read_pk('./final_data/multiesc_predicted_strategy.pk')
    #     print("acc: ", accuracy_score(gt_strategy, predict_strategy))

    predict_strategy_index = 0
    for case_example in data:
        dialog = case_example['dialog']
        dialog_len = len(dialog)
        emotion_type = case_example['emotion_type']
        problem_type = case_example['problem_type']
        situation = case_example['situation']
        tot_strategy = []
        for index, tmp_dic in enumerate(dialog):
            if tmp_dic['speaker'] == 'sys' and tmp_dic['strategy'] != "Others":
                # todo: doesnt it mess with data?
                tot_strategy.append(norm_strategy(tmp_dic['strategy']))

        # initial history is emotion_type + problem_type + situation
        history = [_norm(emotion_type) + sep_token + _norm(
            problem_type) + sep_token + _norm(situation)]

        tmp_strategy_list = []
        for index, tmp_dic in enumerate(dialog):
            text = _norm(tmp_dic['text'])
            if index == 0 and tmp_dic['speaker'] != 'sys':
                # todo: why prepend it to history?
                # handling when conversation starts with help seeker
                history[0] = "<seeker> " + text + sep_token + history[0]
                continue
            if tmp_dic['speaker'] == 'sys' and tmp_dic['strategy'] != "Others":
                # build one example from history up to this point of conversation
                tmp_strategy = norm_strategy(tmp_dic['strategy'])
                save_s = [x for x in tot_strategy[len(tmp_strategy_list):]].copy()
                assert len(save_s) > 0, print(tot_strategy, tmp_strategy_list)
                tmp_history = copy.deepcopy(history)
                if joint_strategy_utt:
                    response = tmp_strategy + " " + text
                else:
                    response = text

                # todo: check if required later:
                # if with_strategy and self.model_type > 4:
                #     if 'test' in file_path:
                #         # tmp_history[-1] = tmp_history[-1] + self.sep_token + predict_strategy[predict_strategy_index]
                #         if self.model_type == 8:
                #             # response = predict_strategy[predict_strategy_index] + " " + text
                #             tmp_history.append(predict_strategy[predict_strategy_index])
                #         else:
                #             tmp_history[-1] = tmp_history[-1] + self.sep_token + predict_strategy[
                #                 predict_strategy_index]
                #         predict_strategy_index += 1
                #     else:
                #         if self.model_type == 8:
                #             tmp_history.append(tmp_strategy)
                #             # response = tmp_strategy + " " + text
                #         else:
                #             tmp_history[-1] = tmp_history[-1] + self.sep_token + tmp_strategy

                total_data.append({
                    "history": tmp_history,
                    "strategy": tmp_strategy,
                    "history_strategy": tmp_strategy_list,
                    "response": response,
                    "future_strategy": ' '.join(save_s),
                    # todo: what is the use case for stage of conversation?
                    "stage": 5 * index // dialog_len,
                })
                tmp_strategy_list.append(tmp_strategy)

            if tmp_dic['speaker'] == 'sys':
                # handling helper utterance with strategy
                tmp_strategy = norm_strategy(tmp_dic['strategy'])
                if with_strategy:
                    # add strategy as control code to the history text
                    tmp_sen = "<helper> " + tmp_strategy + " " + text
                    history.append(tmp_sen)
                else:
                    history.append("<helper> " + text)
            else:
                # help seeker utterance
                utt = text

                if add_cause:
                    cause = tmp_dic['cause']
                    if cause is not None and len(cause) > 0:
                        utt = cause + sep_token + utt

                history.append("<seeker> " + utt)

    # random_idx = random.randint(0, len(total_data) - 1)
    # print(f'printing training example {random_idx}:')
    # print(total_data[random_idx])

    write_json(cached_preprocessed_file, total_data)
    return cached_preprocessed_file

class InputPreprocessor:
    def __init__(
            self,
            preprocessor_type: str,
            tokenizer: PreTrainedTokenizer = None,
            max_source_length: int = None,
            max_target_length: int = None,
    ):
        self.preprocessor_type = preprocessor_type
        self.tokenizer = tokenizer
        self.max_source_length = max_source_length
        self.max_target_length = max_target_length

        processing_functions = {
            'joint_strategy_utterance_generation': self.joint_strategy_utterance_generation,
            'strategy_generation': self.strategy_generation_tokenization,
        }

        self.preprocess = processing_functions[preprocessor_type]

    def strategy_generation_tokenization(self, example):
        history = example['history']
        target = example['future_strategy']
        full_text = self.tokenizer.sep_token.join(history)

        inputs = self.tokenizer(full_text, add_special_tokens=True, max_length=self.max_source_length, truncation=True)
        labels = self.tokenizer(target, add_special_tokens=True, max_length=self.max_target_length, truncation=True)

        return {
            'input_ids': inputs['input_ids'],
            'attention_mask': inputs['attention_mask'],
            'labels': labels['input_ids'],
        }

    def joint_strategy_utterance_generation(self, example):
        history = example['history']
        target = example['response']
        full_text = self.tokenizer.bos_token + " ".join(history) + " <helper> "

        inputs = self.tokenizer(full_text, add_special_tokens=False, max_length=self.max_source_length, truncation=True)
        labels = self.tokenizer(target, add_special_tokens=True, max_length=self.max_target_length, truncation=True)

        return {
            'input_ids': inputs['input_ids'],
            'attention_mask': inputs['attention_mask'],
            'labels': labels['input_ids'],
        }

# class CustomDataCollator(DataCollator):
#     def __init__(self, tokenizer):
#         self.tokenizer = tokenizer
#
#     def sequence_only_strategy_generation_collator(self, batch):
#         pass

def main(
    base_file_path: str,
    tokenizer_name_or_path: str = 'facebook/bart-base',
    add_cause: bool = False,
    with_strategy: bool = False,
):
    splits = ['train', 'valid', 'test']
    tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path)
    tokenizer.add_special_tokens({'sep_token': '<sep>'})
    for split in splits:
        file_path = os.path.join(base_file_path, f"{split}.json")
        construct_conversational_dataset(
            file_path,
            tokenizer,
            add_cause=add_cause,
            with_strategy=with_strategy,
            load_from_cache=False,
        )

if __name__ == "__main__":
    Fire(main)


import warnings
from datetime import datetime

from django.core.cache import cache
from django.urls import reverse
from django.utils.translation import gettext_noop

import dateutil
from memoized import memoized
from corehq.apps.reports.forms import EmailReportForm

from dimagi.utils.dates import DateSpan

from corehq.apps.casegroups.models import CommCareCaseGroup
from corehq.apps.groups.models import Group
from corehq.apps.reports import util
from corehq.apps.reports.dispatcher import (
    CustomProjectReportDispatcher,
    ProjectReportDispatcher,
)
from corehq.apps.reports.exceptions import BadRequestError
from corehq.apps.reports.filters.select import MonthFilter, YearFilter
from corehq.apps.reports.filters.users import UserTypeFilter
from corehq.apps.reports.generic import GenericReportView
from corehq.apps.reports.models import HQUserType
from corehq.apps.users.models import CommCareUser
from corehq.util.timezones.conversions import ServerTime

class ProjectReport(GenericReportView):
    # overriding properties from GenericReportView
    section_name = gettext_noop("Project Reports")
    base_template = 'reports/base_template.html'
    dispatcher = ProjectReportDispatcher
    asynchronous = True

    @property
    def default_report_url(self):
        return reverse('reports_home', args=[self.request.project.name])

    @property
    def template_context(self):
        context = super().template_context
        context.update({
            'user_types': HQUserType.human_readable,
            'email_form': EmailReportForm()
        })
        return context

class CustomProjectReport(ProjectReport):
    dispatcher = CustomProjectReportDispatcher
    emailable = True
    languages = None

    @classmethod
    def get_supports_translations(cls):
        return bool(cls.languages)

class CommCareUserMemoizer(object):

    @memoized
    def by_domain(self, domain, is_active=True):
        users = CommCareUser.by_domain(domain, is_active=is_active)
        for user in users:
            # put users in the cache for get_by_user_id
            # so that function never has to touch the database
            self.get_by_user_id.get_cache(self)[(self, user.user_id)] = user
        return users

    @memoized
    def get_by_user_id(self, user_id):
        return CommCareUser.get_by_user_id(user_id)

class ProjectReportParametersMixin(object):
    """
    All the parameters necessary for the project reports.
    Intended to be mixed in with a GenericReportView object.
    """

    default_case_type = None
    filter_group_name = None
    filter_users_field_class = UserTypeFilter
    include_inactive = False

    # set this to set the report's user ids from within the report
    # (i.e. based on a filter's return value).
    override_user_ids = None

    @property
    @memoized
    def CommCareUser(self):
        return CommCareUserMemoizer()

    @memoized
    def get_all_users_by_domain(self, group=None, user_ids=None, user_filter=None, simplified=False):
        return list(util.get_all_users_by_domain(
            domain=self.domain,
            group=group,
            user_ids=user_ids,
            user_filter=user_filter,
            simplified=simplified,
            CommCareUser=self.CommCareUser
        ))

    @property
    @memoized
    def user_filter(self):
        return self.filter_users_field_class.get_user_filter(self.request)[0]

    @property
    @memoized
    def default_user_filter(self):
        return self.filter_users_field_class.get_user_filter(None)[0]

    @property
    def group_id(self):
        return self.request.GET.get('group', '')

    @property
    @memoized
    def group(self):
        return Group.get(self.group_id) if self.group_id else None

    @property
    def individual(self):
        """
            todo: remember this: if self.individual and self.users:
            self.name = "%s for %s" % (self.name, self.users[0].raw_username)
        """
        return self.request_params.get('individual', '')

    @property
    def mobile_worker_ids(self):
        ids = self.request.GET.getlist('select_mw')
        if '_all' in ids or self.request.GET.get('all_mws', 'off') == 'on':
            cache_str = "mw_ids:%s" % self.domain
            ids = cache.get(cache_str)
            if not ids:
                cc_users = CommCareUser.by_domain(self.domain)
                if self.include_inactive:
                    cc_users += CommCareUser.by_domain(self.domain, is_active=False)
                ids = [ccu._id for ccu in cc_users]
                cache.set(cache_str, ids, 24 * 60 * 60)
        return ids

    @property
    @memoized
    def users(self):
        warnings.warn('Usage of this property is deprecated due to poor performance.', DeprecationWarning)
        if self.filter_group_name and not (self.group_id or self.individual):
            group = Group.by_name(self.domain, self.filter_group_name)
        else:
            group = self.group

        if self.override_user_ids is not None:
            user_ids = self.override_user_ids
        else:
            user_ids = [self.individual]

        return self.get_all_users_by_domain(
            group=group,
            user_ids=tuple(user_ids),
            user_filter=tuple(self.user_filter),
            simplified=True
        )

    @property
    @memoized
    def user_ids(self):
        return [user.user_id for user in self.users]

    @property
    def history(self):
        history = self.request_params.get('history', '')
        if history:
            try:
                return dateutil.parser.parse(history)
            except ValueError:
                pass

    @property
    def case_type(self):
        return self.default_case_type or self.request_params.get('case_type', '')

    @property
    def case_types(self):
        return [_f for _f in self.request.GET.getlist('case_type') if _f]

    @property
    def case_status(self):
        from corehq.apps.reports.filters.select import SelectOpenCloseFilter
        return self.request_params.get(SelectOpenCloseFilter.slug, '')

    @property
    def case_group_ids(self):
        return [_f for _f in self.request.GET.getlist('case_group') if _f]

    @property
    @memoized
    def case_groups(self):
        return [CommCareCaseGroup.get(g) for g in self.case_group_ids]

    @property
    @memoized
    def cases_by_case_group(self):
        case_ids = []
        for group in self.case_groups:
            case_ids.extend(group.cases)
        return case_ids

class CouchCachedReportMixin(object):
    """
        Use this mixin for caching reports as objects in couch.
    """
    _cached_report = None

    @property
    def cached_report(self):
        if not self._cached_report:
            self._cached_report = self.fetch_cached_report()
        return self._cached_report

    def fetch_cached_report(self):
        """
            Here's where you generate your cached report.
        """
        raise NotImplementedError

class DatespanMixin(object):
    """
        Use this where you'd like to include the datespan field.
    """
    datespan_field = 'corehq.apps.reports.filters.dates.DatespanFilter'
    datespan_default_days = 7
    datespan_max_days = None
    inclusive = True
    default_datespan_end_date_to_today = False

    _datespan = None

    @property
    def datespan(self):
        if self._datespan is None:
            datespan = self.default_datespan
            if self.request.datespan.is_valid() and not self.request.datespan.is_default:
                datespan.enddate = self.request.datespan.enddate
                datespan.startdate = self.request.datespan.startdate
                datespan.is_default = False
            elif self.request.datespan.get_validation_reason() == "You can't use dates earlier than the year 1900":
                raise BadRequestError()
            self.request.datespan = datespan
            # todo: don't update self.context here. find a better place! AGH! Sorry, sorry.
            self.context.update(dict(datespan=datespan))
            self._datespan = datespan
        return self._datespan

    @property
    def default_datespan(self):
        # DateSpan.since() will make enddate default to yesterday when it's None
        enddate = None
        if self.default_datespan_end_date_to_today:
            enddate = ServerTime(datetime.utcnow()).user_time(self.timezone).done().date()

        datespan = DateSpan.since(self.datespan_default_days, enddate=enddate, inclusive=self.inclusive,
            timezone=self.timezone)
        datespan.max_days = self.datespan_max_days
        datespan.is_default = True
        return datespan

class MonthYearMixin(object):
    """
        Similar to DatespanMixin, but works with MonthField and YearField
    """
    fields = [MonthFilter, YearFilter]

    _datespan = None

    @property
    def datespan(self):
        if self._datespan is None:
            datespan = DateSpan.from_month(self.month, self.year)
            self.request.datespan = datespan
            self.context.update(dict(datespan=datespan))
            self._datespan = datespan
        return self._datespan

    @property
    def month(self):
        if 'month' in self.request_params:
            return int(self.request_params['month'])
        else:
            return datetime.utcnow().month

    @property
    def year(self):
        if 'year' in self.request_params:
            return int(self.request_params['year'])
        else:
            return datetime.utcnow().year


# 322. Coin Change
# medium

class Solution(object):
    def coinChange(self, coins, amount):
        """
        :type coins: List[int]
        :type amount: int
        :rtype: int
        """
        coins.sort(reverse=True)
        dp = [amount + 1] * (amount + 1)
        dp[0] = 0
        for i in range(1, amount + 1):
            for coin in coins:
                if coin > i:
                    continue
                leftOver = i - coin
                if dp[leftOver] != -1:
                    dp[i] = min(dp[leftOver] + 1, dp[i])
        return dp[amount] if dp[amount] <= amount else -1


import logging
logging.basicConfig(format="[%(asctime)s] %(message)s", datefmt="%m-%d %H:%M:%S")

import tensorflow as tf
import numpy as np

logger = logging.getLogger(__name__)

def get_shape(layer):
    return layer.get_shape().as_list()

def skew(inputs, scope="skew"):
    with tf.name_scope(scope):
        batch, height, width, channel = get_shape(inputs)
        rows = tf.split(inputs, height, 1)

        new_width = width + height - 1
        new_rows = []

        for idx, row in enumerate(rows):
            transposed_row = tf.transpose(tf.squeeze(row, [1]), [0, 2, 1]) # [batch, channel, width]
            squeezed_row = tf.reshape(transposed_row, [-1, width]) # [batch*channel, width]
            padded_row = tf.pad(squeezed_row, [[0, 0], [idx, height - 1 - idx]]) # [batch*channel, width]

            unsqueezed_row = tf.reshape(padded_row, [-1, channel, new_width]) # [batch, channel, new_width]
            untransposed_row = tf.transpose(unsqueezed_row, [0, 2, 1]) # [bacth, new_width, channel]

            assert get_shape(untransposed_row) == [batch, new_width, channel], "wrong shape of skewed row"
            new_rows.append(untransposed_row)

        outputs = tf.stack(new_rows, axis=1, name="output")
        assert get_shape(outputs) == [batch, height, new_width, channel], "wrong shape of outputs"

    logger.debug('[skew] %s : %s %s -> %s %s' \
        % (scope, inputs.name, inputs.get_shape(), outputs.name, outputs.get_shape()))
    return outputs

def unskew(inputs, width=None, scope="unskew"):
    with tf.name_scope(scope):
        batch, height, skewed_width, channel = get_shape(inputs)
        width = width if width else height

        new_rows = []
        rows = tf.split(inputs, height, 1)

        for idx, row in enumerate(rows):
            new_rows.append(tf.slice(row, [0, 0, idx, 0], [-1, -1, skewed_width - width + 1,-1]))
        # outputs = tf.stack(new_rows, axis=1, name="output")
        outputs = tf.concat(new_rows, axis=1, name="output")
    logger.debug('[unskew] %s : %s %s -> %s %s'\
        % (scope, inputs.name, inputs.get_shape(), outputs.name, outputs.get_shape()))
    return outputs

WEIGHT_INITIALIZER = tf.contrib.layers.xavier_initializer()

def conv2d(
    inputs,
    num_outputs,
    kernel_shape, # [kernel_height, kernel_width]
    mask_type, # None, "A" or "B",
    strides=[1, 1], # [column_wise_stride, row_wise_stride]
    padding="SAME",
    activation_fn=None,
    weights_initializer=WEIGHT_INITIALIZER,
    weights_regularizer=None,
    biases_initilizer=tf.zeros_initializer(),
    biases_regularizer=None,
    scope="conv2d"):
    with tf.variable_scope(scope):
        mask_type = mask_type.lower()
        batch, height, width, channel = inputs.get_shape().as_list()

        kernel_h, kernel_w = kernel_shape
        stride_h, stride_w = strides

        assert kernel_h % 2 == 1 and kernel_w % 2 == 1, \
        "kernel height and width should be odd number"

        center_h = kernel_h // 2
        center_w = kernel_w // 2

        weights_shape = [kernel_h, kernel_w, channel, num_outputs]
        weights = tf.get_variable("weights", weights_shape,
        tf.float32, weights_initializer, weights_regularizer)

        if mask_type is not None:
            mask = np.ones((kernel_h, kernel_w, channel, num_outputs), dtype=np.float32)
            mask[center_h, center_w+1:,:, :] = 0.
            mask[center_h+1, :, :, :] = 0.

            if mask_type == 'a':
                mask[center_h, center_w, :, :] = 0.

            weights *= tf.constant(mask, dtype=tf.float32)
            tf.add_to_collection('conv2d_weights_%s' % mask_type, weights)

        outputs = tf.nn.conv2d(inputs, weights, [1, stride_h, stride_w, 1], padding=padding, name='outputs')
        tf.add_to_collection('conv2d_outputs', outputs)

        if biases_initilizer != None:
            biases = tf.get_variable("biases", [num_outputs,],
                tf.float32, biases_initilizer, biases_regularizer)
            outputs = tf.nn.bias_add(outputs, biases, name='outputs_plus_b')

        if activation_fn:
            outputs = activation_fn(outputs, name='outputs_with_fn')

        logger.debug('[conv2d_%s] %s : %s %s -> %s %s' \
            % (mask_type, scope, inputs.name, inputs.get_shape(), outputs.name, outputs.get_shape()))

        return outputs


import json
import logging
import configparser
import subprocess
import sys

ALIAS = sys.argv[1]
RESOURCE_ROLE = sys.argv[2]

cfg = configparser.ConfigParser()
cfg.read('deployment_config.ini')

# parse lambda configuration parameters
CONFIG_PARAMS = {}
for config_tuple in cfg.items('configuration'):
    name = config_tuple[0]
    val = config_tuple[1]
    CONFIG_PARAMS[name] = val

# https://stackoverflow.com/questions/5466451/how-can-i-print-literal-curly-brace-characters-in-python-string-and-also-use-fo#5466478
VARS_STR_TEMPLATE = "Variables={{{}}}"
env_vars = ""
for config_tuple in cfg.items('environment'):
    env_vars += "{k}={v},".format(k=config_tuple[0], v=config_tuple[1])

if env_vars.endswith(","):
    env_vars = env_vars[:-1]
VARS_STR = VARS_STR_TEMPLATE.format(env_vars)
print("Parsed environment variables:\n{}".format(VARS_STR))

def is_func_new(funcname):
    """
    determine if function being deployed is brand new or just needs updates
    """
    bashCommand = "aws lambda get-function \
    --function-name {fname}".format(
        fname=funcname
    )

    try:
        subprocess.check_output(bashCommand.split())
    except subprocess.CalledProcessError as e:
        # if the error is raised it means the functions does not exist
        logging.error(e)
        print("returning True")
        return True
    print("returning False")
    return False

def deploy_lambda(new):
    """
    ISNEW=$1
    RESOURCE_ROLE=$2
    ALIAS=$3

    FNAME=$4
    HANDLER=$5
    TIMEOUT=$6
    MEMSIZE=$7
    DESC=$8
    ENV=$9
    RUNTIME=${10}
    REGION=${11}
    """
    bashCommand = "bash scripts/aws-lambda-deploy.sh \
    {isnew} \
    {AWS_LAMBDA_ROLE} \
    {alias} \
    {funcname} {handler} {timeout} {memsize} '{desc}' '{env}' '{runtime}' '{region}'".format(
        AWS_LAMBDA_ROLE=RESOURCE_ROLE,
        isnew=new,
        alias=ALIAS,
        funcname=CONFIG_PARAMS['function-name'],
        desc=CONFIG_PARAMS['description'],
        runtime=CONFIG_PARAMS['runtime'],
        handler=CONFIG_PARAMS['handler'],
        region=CONFIG_PARAMS['region'],
        timeout=CONFIG_PARAMS['timeout'],
        memsize=CONFIG_PARAMS['memory-size'],
        env=VARS_STR
    )

    process = subprocess.Popen(bashCommand, stdout=subprocess.PIPE, shell=True)
    output, error = process.communicate()
    # print(output)
    if error:
        logging.error(error)

deploy_lambda(new=is_func_new(CONFIG_PARAMS['function-name']))


import os
from shutil import copy

SOURCE = f"//ads.warwick.ac.uk/shared/HCSS6/Shared305/Microscopy/Jeffrey-Ede/models/recurrent_conv-1/"
SINK = "//Desktop-sa1evjv/h/sink/recurrent_conv-1/"

WHITELIST = [str(i) for i in range(98, 144)]

for item in os.listdir(SOURCE):
    path = SOURCE+item
    if os.path.isdir(item) and item in WHITELIST:
        os.makedirs(SINK+item)
        for subitem in os.listdir(path):
            subpath = path + "/" + subitem
            dst = SINK + item + "/" + subitem
            if os.path.isdir(subpath):
                os.makedirs(dst)
                if subitem == "dnc":
                    for file in os.listdir(subpath):
                        if not file == "__pycache__":
                            source_loc = subpath + "/" + file
                            sink_loc = dst + "/" + file
                            copy(source_loc, sink_loc)
            elif subitem.split(".")[-1] in ["py"] or subitem == "notes.txt":
                copy(subpath, dst)

    elif os.path.isfile(path):
        try:
            copy(path, SINK+item)
        except:
            continue


import pandas as pd

def change_file(file_name, file_type, var_year):
    df = pd.read_parquet(f"raw_data/{file_name}{var_year}{file_type}")
    df.to_csv(f"raw_data_csv/{file_name}{var_year}.csv", index=True)
    return

def main():
    # hyper_parameter
    start_year = 2023
    end_year = 2023

    # static_parameter
    file_name = "blue_alliance_"
    file_type = ".parquet.gzip"

    for var_year in range(start_year, end_year + 1):
        try:
            change_file(file_name, file_type, var_year)
        except Exception as E:
            print(E)
    return

if __name__ == "__main__":
    main()


import csv
import math
import sys

import pss
import psqi

# indexes of field names
timestamp = 0
grade = 1
exercise = 31
screen_time = 32
extracurr = 33

# validate cmdline arguments
# i know it's not robust, but i probably shouldn't overengineer
# some code for an AP stats class which only I will ever run
if len(sys.argv) < 3:
    print("usage: $ python3 score.py input.csv output.csv [--no-print]")
    sys.exit()

printing = True
if len(sys.argv) == 4 and sys.argv[3] == '--no-print':
    printing = False

pss_scores = []
psqi_scores = []

# main routine
with open(sys.argv[1], "r", encoding='utf8') as fin:
    reader = csv.reader(fin)
    next(reader)

    with open(sys.argv[2], "w", encoding='utf8') as fout:
        writer = csv.writer(fout)

        # write output field names
        fout.write("Timestamp,Grade,PSS,PSQI,Exercise,Screen Time,Extracurriculars\n")

        print("**************************************")
        for response in reader:
            pss_score = pss.calc_pss(response)
            psqi_score = psqi.calc_psqi(response, printing)

            pss_scores.append(pss_score)
            psqi_scores.append(psqi_score)

            # if printing:
            print('| ' + response[timestamp], end='')
            print(f"  PSS {pss_score:02d}", end='')
            print(f"  PSQI {psqi_score:02d} |")
            print("**************************************")

            row = [
                response[timestamp], response[grade],
                pss_score, psqi_score, response[exercise],
                response[screen_time], response[extracurr]
                ]
            writer.writerow(row)

def mean(ls):
    total = 0
    for n in ls:
        total += n
    return total / len(ls)

def std_dev(ls, mean):
    total = 0
    for n in ls:
        total += ((n - mean) ** 2)
    var = total / (len(ls) - 1)
    return math.sqrt(var)

print(f"\n\nSCORING COMPLETE.")
print(f"{len(pss_scores):d} SUBJECTS ANALYZED.\n")

mean_psqi = mean(psqi_scores)
mean_pss = mean(pss_scores)

print(f"MEAN    PSQI  {mean_psqi:.02f}")
print(f"MEAN    PSS   {mean_pss:.02f}")
print(f"STD_DEV PSQI  {std_dev(psqi_scores, mean_psqi):.02f}")
print(f"STD_DEV PSS   {std_dev(pss_scores, mean_pss):.02f}")


import os

class KeyboardManager:
	def __init__(self,config_file = None):
		"""
		creates keyboard manager. It will take in key lists, and return what keys have been pressed in the meantime.
		It will translate pressed keys to a set of values, with which it is initalisted, say "W" to "up"
		Initialisation creates default config, or uses one given (config_file) as filepath
		config_file -> file to be read config from
		"""
		self.config = {}
		self.config["KEY_UP"] = "up"
		self.config["KEY_DOWN"] = "down"
		self.config["KEY_LEFT"] = "left"
		self.config["KEY_RIGHT"] = "right"
		#self.config["s_up"] = "KEY_UP"
		#self.config["s_down"] = "KEY_DOWN"
		#self.config["s_left"] = "KEY_LEFT"
		#self.config["s_right"] = "KEY_RIGHT"
		self.config[" "] = "shoot"
		self.config["E"] = "special"

		self.state = {}
		for i in self.config.values():
			self.state[i] = False

		if config_file != None:
			self.read_config(config_file)

	def read_config(self,config_file):
		"""
 		Reads confing file. File consists of lines so that name=key
		Called at initialisation if config is given
		"""
		f = open(config_file, 'r')
		lines = f.readlines()
		for l in lines:
			k,v=l.split("=")
			v=v[0:-1]
			if v in self.config.keys():
				self.config[v]=k
			else:
				raise ValueError("Keyboard configuration: key: "+v+" not found in registry")

	def write_config(self,target_filename):
		"""
 		Writes config to given filename
		"""
		if os.path.isfile(target_filename):
			os.remove(target_filename)
		f = open(target_filename,"w")
		for k,v in self.config.items():
			f.write(v+"="+k+"\n")

	def poll(self,v_key):
		"""
 		Returns True if v_key (key name) is pressed.
		v_key -> string, key name. Not key as "W" or "KEY_UP" but rather "up" or "shoot"
		"""
		if v_key in self.state.keys():
			return self.state[v_key]
		else:
			return None

	def get_keys(self):
		"""
		returns dictionary of currently pressed key names (shoot/up etc)
		"""
		return self.state

	def on_keys(self,key_list):
		"""
 		Takes in a list of keys pressed in last update, returns change list.
		Returned change list = [(string k_name,boolean val),(...)...] where k_name is shoot/up etc and val = True if pressed or False if released

		key_list -> list of key input ("W","KEY_UP" etc) from curses.
		"""
		setlist = list(set(key_list))
		v_list = [self.config[k] for k in setlist if k in self.config.keys()]
		emitted_changes = []

		for k,v in self.state.items():
			if k in v_list and v == False:
				self.state[k] = True
				emitted_changes.append((k,True))
			elif (not (k in v_list)) and v == True:
				self.state[k] = False
				emitted_changes.append((k,False))

		return emitted_changes


from dirs import *

f = open("puzzle_input.txt", "r")
dirs = parse_directories(f)
f.close()

disk_space = 70000000
needed_space = 30000000
used_disk = compute_dir_size("/", dirs)
free_disk = disk_space - used_disk

smallest = used_disk
for dir in dirs.keys():
    dir_sum = compute_dir_size(dir, dirs)

    if free_disk + dir_sum >= needed_space and dir_sum < smallest:
        smallest = dir_sum

print(smallest) # 5756764