import pygame
import random
import time

# initialize pygame
pygame.init()

# set up the window
WIDTH = 800
HEIGHT = 600
window = pygame.display.set_mode((WIDTH, HEIGHT))
pygame.display.set_caption("Type Racers")

# set up the fonts
font = pygame.font.SysFont("comicsansms", 32)
font_small = pygame.font.SysFont("comicsansms", 16)

# set up the colors
WHITE = (255, 255, 255)
BLACK = (0, 0, 0)

# set up the game clock
clock = pygame.time.Clock()

# load the words from the file
with open("words.txt", "r") as f:
    words = f.readlines()
    words = [word.strip() for word in words]

# set up the game
class Game:
    def __init__(self):
        self.word = random.choice(words)
        self.word_index = 0
        self.input_text = ""
        self.start_time = None
        self.elapsed_time = None
        self.is_running = False
        self.is_finished = False

    def start(self):
        self.is_running = True
        self.start_time = time.time()

    def stop(self):
        self.is_running = False
        self.elapsed_time = round(time.time() - self.start_time, 2)
        self.is_finished = True

    def get_input(self, event):
        if event.key == pygame.K_BACKSPACE:
            self.input_text = self.input_text[:-1]
        elif event.key == pygame.K_RETURN:
            if self.input_text == self.word:
                self.stop()
            else:
                self.input_text = ""
        elif event.unicode.isalpha():
            self.input_text += event.unicode

    def update(self):
        if self.is_running:
            if self.word_index == len(self.word):
                self.stop()
            elif self.input_text == self.word[:self.word_index+1]:
                self.word_index += 1

    def render(self):
        window.fill(WHITE)

        # display the word to type
        word_text = font.render(self.word, True, BLACK)
        window.blit(word_text, (50, 100))

        # display the user's input
        input_text = font.render(self.input_text, True, BLACK)
        window.blit(input_text, (50, 150))

        # display the time elapsed
        if self.is_running:
            elapsed_time = round(time.time() - self.start_time, 2)
        else:
            elapsed_time = self.elapsed_time
        time_text = font_small.render("Time: " + str(elapsed_time), True, BLACK)
        window.blit(time_text, (WIDTH - 100, 50))

        # display the game over message
        if self.is_finished:
            if self.input_text == self.word:
                game_over_text = font.render("You win!", True, BLACK)
            else:
                game_over_text = font.render("You lose!", True, BLACK)
            window.blit(game_over_text, (50, 250))

        pygame.display.update()

game = Game()

# run the game loop
while True:
    # handle events
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            pygame.quit()
            quit()
        elif event.type == pygame.KEYDOWN:
            game.get_input(event)


# -*- coding: utf_8 -*-
"""Module for strings-method for java."""
import logging
import os

from androguard.core.bytecodes import apk

from mobsf.StaticAnalyzer.views.shared_func import (
    is_secret,
    url_n_email_extract,
)

logger = logging.getLogger(__name__)

def strings_from_apk(app_file, app_dir, elf_strings):
    """Extract the strings from an app."""
    try:
        logger.info('Extracting Strings from APK')
        dat = []
        secrets = []
        urls = []
        urls_nf = []
        emails_nf = []
        apk_file = os.path.join(app_dir, app_file)
        and_a = apk.APK(apk_file)
        rsrc = and_a.get_android_resources()
        if rsrc:
            pkg = rsrc.get_packages_names()[0]
            rsrc.get_strings_resources()
            for i in rsrc.values[pkg].keys():
                res_string = rsrc.values[pkg][i].get('string')
                if res_string:
                    for duo in res_string:
                        cap_str = '"' + duo[0] + '" : "' + duo[1] + '"'
                        if is_secret(duo[0] + '"'):
                            secrets.append(cap_str)
                        dat.append(cap_str)
            data_string = ''.join(dat)
            urls, urls_nf, emails_nf = url_n_email_extract(
                data_string, 'Android String Resource')
        if elf_strings:
            for solib in elf_strings:
                for so, str_list in solib.items():
                    # add to strings from jar
                    dat.extend(str_list)
                    # extract url, email
                    so_str = ' '.join(str_list)
                    su, suf, sem = url_n_email_extract(
                        so_str, so)
                    urls.extend(su)
                    urls_nf.extend(suf)
                    emails_nf.extend(sem)
        strings_dat = list(set(dat))
        return {
            'strings': strings_dat,
            'urls_list': urls,
            'url_nf': urls_nf,
            'emails_nf': emails_nf,
            'secrets': secrets,
        }
    except Exception:
        logger.exception('Extracting Strings from APK')
        return {}


from estate import Estate
from thoroughfare import Thoroughfare
from property import Property
from household import Household
from receipt import Receipt
from invoice import Invoice
from typing import List

class Basic_User:

    def __init__(self, estate:Estate):
        self.__estate = estate

    def get_estate(self) ->Estate:
        return self.__estate

### 1i ####
    def create_thoroughfare(self)->None:
        name = input("What is the name of the thoroughfare?: ")
        self.get_estate().create_thoroughfare(name, [])

    def view_thoroughfare(self, position:int)->Thoroughfare:
        self.get_estate().view_thoroughfare(position)

    def update_thoroughfare(self, position:int)->None:
        new_name = input("Enter updated thoroughfare name: ")
        self.get_estate().get_thoroughfare(position).set_name(new_name)

    def create_full_thoroughfare(self)->None:
        try:
            name_tf = input("What is the name of this thoroughfare?: ")
            TF = self.get_estate().create_thoroughfare_new(name_tf)
            amount_properties = int(input("How much properties would you like to add?: "))
            for x in range (amount_properties):
                print("Entering details for property number " + str(x+1))
                prop_type = input("What is the property type? :")
                address = input("What is the address?: ")
                completion_date = input("What is the completion date?: ")
                mgmt_type = input("What is the management type?: ")
                owner = input("Who is the custodian of the property?: ")

                Prop = TF.create_property_new(prop_type, [], address, owner, completion_date, mgmt_type)

                amount_households = int(input("How many households are in the property?: "))
                for x in range (amount_households):
                    print("Entering details for household: " +str(x+1))
                    Prop.create_household_new()
        except:
            print("Please ensure that the number of properties and households is an integer entry only.")

### 1ii ###

    def create_property(self, position:int)->None:
        self.get_estate().get_thoroughfare(position).create_property()

    def view_property(self, tf_position:int, p_position:int)->None:
        self.get_estate().get_thoroughfare(tf_position).view_property(p_position)

    def update_property(self, tf_position:int, p_position:int)->None:
        self.get_estate().get_thoroughfare(tf_position).update_property_new(p_position)

    def create_full_property(self, tf_position:int)->None:
        try:
            TF = self.get_estate().get_thoroughfare(tf_position)
            amount_properties = int(input("How many properties would you like to add?: "))
            for x in range (amount_properties):
                    print("Entering details for property number " + str(x+1))
                    prop_type = input("What is the property type? :")
                    address = input("What is the address?: ")
                    completion_date = input("What is the completion date?: ")
                    mgmt_type = input("What is the management type?: ")
                    owner = input("Who is the custodian of the property?: ")

                    Prop = TF.create_property_new(prop_type, [], address, owner, completion_date, mgmt_type)

                    amount_households = int(input("How many households are in the property?: "))
                    for x in range (amount_households):
                        print("Entering details for household: " +str(x+1))
                        Prop.create_household_new()
        except:
            print("Please ensure that the number of properties and households is an integer entry only.")

### 1iii ###

    def create_household(self, tf_position:int, p_position:int)->None:
        self.get_estate().get_thoroughfare(tf_position).get_property(p_position).create_household()

    def view_household(self, tf_position:int, p_position:int, h_position:int)->None:
        self.get_estate().get_thoroughfare(tf_position).get_property(p_position).view_household(h_position)

    def update_household(self,tf_position:int, p_position:int, h_position:int)->None:
        new_name = input("What is the name of the custodian?: ")
        self.get_estate().get_thoroughfare(tf_position).get_property(p_position).get_object_in_household_list(h_position).set_custodian(new_name)

    def create_full_household(self, tf_position:int, property_position:int)->None:

        try:
            Prop = self.get_estate().get_thoroughfare(tf_position).get_property(property_position)
            amount_households = int(input("How many households would you like to add?: "))
            for x in range(amount_households):
                print("Entering details for household: " + str(x + 1))
                Prop.create_household_new()
        except:
            print("Please ensure that the number of properties and households is an integer entry only.")

### 1iv ####

    def generate_invoice_specific_household(self,tf_position:int, p_position:int, h_position:int)->None:
        try:
            payment_date = input("What is the payment date?: ")
            total_value = int(input("What is the total value of the invoice?: "))
            amount_paid = int(input("What is the payment being made today?: "))
            self.get_estate().get_thoroughfare(tf_position).get_property(p_position).get_object_in_household_list(
                h_position).create_invoice(payment_date,total_value,amount_paid)
        except:
            print("Please ensure total_value and amount_paid are both integer entries")

### 1v ###

    def take_payment_generate_receipt(self,tf_position:int, p_position:int, h_position:int)->None:
        try:
            invoice_position = int(input("What is the position of the invoice?: "))
            payment_date = input("What is the payment date?: ")
            amount_paid = int(input("What is the payment being made today?: "))
            self.get_estate().get_thoroughfare(tf_position).get_property(p_position).get_object_in_household_list(
                h_position).take_payment_generate_receipt(payment_date, amount_paid,invoice_position)
        except:
            print("Please ensure amount paid is an integer value")

### 1vi ###

    def print_invoices_and_receipts_for_household(self,tf_position:int, p_position:int, h_position:int):
        self.get_estate().get_thoroughfare(tf_position).get_property(p_position).get_object_in_household_list(
            h_position).print_invoices()
        self.get_estate().get_thoroughfare(tf_position).get_property(p_position).get_object_in_household_list(
            h_position).print_receipts()


from selenium import webdriver
from selenium.webdriver.common.by import By

chrome_options = webdriver.ChromeOptions()
chrome_options.add_experimental_option("detach", True)

driver = webdriver.Chrome(options=chrome_options)
driver.get("https://www.python.org")

events = {}
for i in range(1, 6):
    time = driver.find_element(By.XPATH, value=f'//*[@id="content"]/div/section/div[2]/div[2]/div/ul/li[{i}]/time').get_attribute("datetime").split("T")[0]
    name = driver.find_element(By.XPATH, value=f'//*[@id="content"]/div/section/div[2]/div[2]/div/ul/li[{i}]/a').text
    events[i-1] = {
        "time": time,
        "name": name
    }

driver.quit()
print(events)


strin=str(input())
lis=['d','h','o','n','i']
cut=0
for i in range(len(strin)):
    if(strin[i] in lis):
        cut=cut+1
if(cut-len(strin)==0):
    print("yes")
else:
    print("no")


from pyamaze import maze, agent, textLabel
from solver import *
import sys

sys.setrecursionlimit(10000)

sizex = 50
sizey = 50
m = maze(sizex, sizey)
start = (sizex, sizey)
end = (1, 1)
m.CreateMaze(end[0], end[1])

#Graph Making
mazeGraph = graph.generateGraph(m.maze_map)

#Pointer
a=agent(m,start[0],start[1], filled=True, footprints=True,color='red')
r=agent(m,start[0],start[1], filled=True, footprints=True,color='green')

#Path Making
path = solver.dfsCaller(mazeGraph, start, end)

#Counters
cellsChecked = textLabel(m, 'Cells Checked', len(solver.cPath))
cellsOnPath = textLabel(m, 'Path Cells', len(path))

#Path Drawing
m.tracePath({a:solver.cPath},delay=10)
m.tracePath({r:path},delay=10)

#Run
m.run()


#!/usr/bin/env python3

import subprocess
import random
import datetime

subprocess.run(["make", "parallel"], universal_newlines=True)
subprocess.run(["make", "serial"], universal_newlines=True)

print("n,p,s")

for i in range(1, 17):

	# RUN PARALLEL #
	start = datetime.datetime.now()
	p_output = subprocess.run(["mpirun", "-np", "12", "./nqueens", str(i), "0", "1"], stdout=subprocess.PIPE, universal_newlines=True).stdout
	end = datetime.datetime.now()
	parallel_runtime = end - start

	# RUN SERIAL #
	start = datetime.datetime.now()
	s_output = subprocess.run(["./serial", str(i)], stdout=subprocess.PIPE, universal_newlines=True).stdout
	end = datetime.datetime.now()
	serial_runtime = end - start

	print(str(i) + "," + str(parallel_runtime.total_seconds()) + "," + str(serial_runtime.total_seconds()))


from bs4 import BeautifulSoup
import requests

data = requests.get("https://gogoanime.gr/genre/action")
converted_data = data.content
soup = BeautifulSoup(converted_data , 'lxml')

names = soup.find_all("p",{"class": "name"})
# images = soup.find_all("div",{"class": "img"})

for image in soup.find_all('div', attrs={'class':'img'}):
    for picture in image.find_all('img'):
        print(picture["src"])


import torch
import torch.nn as nn

# NeuralNets
from nnet import preprocessing
from nnet import layers
from nnet import modules
from nnet import blocks
from nnet import attentions
from nnet import transforms
from nnet import normalizations

###############################################################################
# Networks
###############################################################################

class ResNet(nn.Module):

    """ ResNet (ResNet18, ResNet34, ResNet50, ResNet101, ResNet152)

    Models: 224 x 224
    ResNet18: 11,689,512 Params
    ResNet34: 21,797,672 Params
    ResNet50: 25,557,032 Params
    ResNet101: 44,549,160 Params
    Resnet152: 60,192,808 Params

    Reference: "Deep Residual Learning for Image Recognition" by He et al.
    https://arxiv.org/abs/1512.03385

    """

    def __init__(self, dim_input=3, dim_output=1000, model="ResNet50", include_stem=True, include_head=True):
        super(ResNet, self).__init__()

        assert model in ["ResNet18", "ResNet34", "ResNet50", "ResNet101", "ResNet152"]

        if model == "ResNet18":
            dim_stem = 64
            dim_blocks = [64, 128, 256, 512]
            num_blocks = [2, 2, 2, 2]
            bottleneck = False
        elif model == "ResNet34":
            dim_stem = 64
            dim_blocks = [64, 128, 256, 512]
            num_blocks = [3, 4, 6, 3]
            bottleneck = False
        elif model == "ResNet50":
            dim_stem = 64
            dim_blocks = [256, 512, 1024, 2048]
            num_blocks = [3, 4, 6, 3]
            bottleneck = True
        elif model == "ResNet101":
            dim_stem = 64
            dim_blocks = [256, 512, 1024, 2048]
            num_blocks = [3, 4, 23, 3]
            bottleneck = True
        elif model == "ResNet152":
            dim_stem = 64
            dim_blocks = [256, 512, 1024, 2048]
            num_blocks = [3, 8, 36, 3]
            bottleneck = True

        self.stem = nn.Sequential(
            layers.Conv2d(in_channels=dim_input, out_channels=dim_stem, kernel_size=(7, 7), stride=(2, 2), weight_init="he_normal", bias=False),
            normalizations.BatchNorm2d(num_features=dim_stem),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
        ) if include_stem else nn.Identity()

        # Blocks
        self.blocks = nn.ModuleList()
        for stage_id in range(4):

            for block_id in range(num_blocks[stage_id]):

                # Projection Block
                if block_id == 0:
                    if stage_id == 0:
                        stride = (1, 1)
                        bottleneck_ratio = 1
                        in_features = dim_stem
                    else:
                        stride = (2, 2)
                        bottleneck_ratio = 2
                        in_features = dim_blocks[stage_id-1]
                # Default Block
                else:
                    stride = (1, 1)
                    in_features = dim_blocks[stage_id]
                    bottleneck_ratio = 4

                if bottleneck:
                    self.blocks.append(blocks.ResNetBottleneckBlock(
                        in_features=in_features,
                        out_features=dim_blocks[stage_id],
                        bottleneck_ratio=bottleneck_ratio,
                        kernel_size=(3, 3),
                        stride=stride,
                        act_fun="ReLU",
                        joined_post_act=True
                    ))
                else:
                    self.blocks.append(blocks.ResNetBlock(
                        in_features=in_features,
                        out_features=dim_blocks[stage_id],
                        kernel_size=(3, 3),
                        stride=stride,
                        act_fun="ReLU",
                        joined_post_act=True
                    ))

        # Head
        self.head = nn.Sequential(
            layers.GlobalAvgPool2d(),
            layers.Linear(in_features=dim_blocks[-1], out_features=dim_output, weight_init="he_normal", bias_init="zeros")
        ) if include_head else nn.Identity()

    def forward(self, x):

        # (B, Din, H, W) -> (B, D0, H//4, W//4)
        x = self.stem(x)

        # (B, D0, H//4, W//4) -> (B, D4, H//32, W//32)
        for block in self.blocks:
            x = block(x)

        # (B, D4, H//32, W//32) -> (B, Dout)
        x = self.head(x)

        return x

class Transformer(nn.Module):

    def __init__(self, dim_model, num_blocks, att_params={"class": "MultiHeadAttention", "params":{"num_heads": 4, "weight_init": "normal_02", "bias_init": "zeros"}}, ff_ratio=4, emb_drop_rate=0.1, drop_rate=0.1, act_fun="GELU", pos_embedding=None, mask=None, inner_dropout=False, weight_init="normal_02", bias_init="zeros", post_norm=False):
        super(Transformer, self).__init__()

        # Positional Embedding
        self.pos_embedding = pos_embedding

        # Input Dropout
        self.dropout = nn.Dropout(p=emb_drop_rate)

        # Mask
        self.mask = mask

        # Transformer Blocks
        self.blocks = nn.ModuleList([blocks.TransformerBlock(
            dim_model=dim_model,
            ff_ratio=ff_ratio,
            att_params=att_params,
            drop_rate=drop_rate,
            inner_dropout=inner_dropout,
            act_fun=act_fun,
            weight_init=weight_init,
            bias_init=bias_init,
            post_norm=post_norm
        ) for block_id in range(num_blocks)])

        # LayerNorm
        self.layernorm = nn.LayerNorm(normalized_shape=dim_model) if not post_norm else nn.Identity()

    def forward(self, x, lengths=None):

        # Pos Embedding
        if self.pos_embedding != None:
            x = self.pos_embedding(x)

        # Input Dropout
        x = self.dropout(x)

        # Mask (1 or B, 1, N, N)
        if self.mask != None:
            mask = self.mask(x, lengths)
        else:
            mask = None

        # Transformer Blocks
        for block in self.blocks:
            x = block(x, mask=mask)

        # LayerNorm
        x = self.layernorm(x)

        return x

class ConformerInterCTC(nn.Module):

    def __init__(self, dim_model, num_blocks, interctc_blocks, vocab_size, loss_prefix="ctc", att_params={"class": "MultiHeadAttention", "num_heads": 4}, conv_params={"class": "Conv1d", "params": {"padding": "same", "kernel_size": 31}}, ff_ratio=4, drop_rate=0.1, pos_embedding=None, mask=None, conv_stride=1, batch_norm=True):
        super(ConformerInterCTC, self).__init__()

        # Inter CTC Params
        self.interctc_blocks = interctc_blocks
        self.loss_prefix = loss_prefix

        # Single Stage
        if isinstance(dim_model, int):
            dim_model = [dim_model]
        if isinstance(num_blocks, int):
            num_blocks = [num_blocks]

        # Positional Embedding
        self.pos_embedding = pos_embedding

        # Input Dropout
        self.dropout = nn.Dropout(p=drop_rate)

        # Mask
        self.mask = mask

        # Conformer Stages
        i = 1
        self.conformer_blocks = nn.ModuleList()
        self.interctc_modules = nn.ModuleList()
        for stage_id in range(len(num_blocks)):

            # Conformer Blocks
            for block_id in range(num_blocks[stage_id]):

                # Transposed Block
                transposed_block = "Transpose" in conv_params["class"]

                # Downsampling Block
                down_block = ((block_id == 0) and (stage_id > 0)) if transposed_block else ((block_id == num_blocks[stage_id] - 1) and (stage_id < len(num_blocks) - 1))

                # Block
                self.conformer_blocks.append(blocks.ConformerBlock(
                    dim_model=dim_model[stage_id - (1 if transposed_block and down_block else 0)],
                    dim_expand=dim_model[stage_id + (1 if not transposed_block and down_block else 0)],
                    ff_ratio=ff_ratio,
                    drop_rate=drop_rate,
                    att_params=att_params[stage_id - (1 if transposed_block and down_block else 0)] if isinstance(att_params, list) else att_params,
                    conv_stride=1 if not down_block else conv_stride[stage_id] if isinstance(conv_stride, list) else conv_stride,
                    conv_params=conv_params[stage_id] if isinstance(conv_params, list) else conv_params,
                    batch_norm=batch_norm
                ))

                # InterCTC Block
                if i in interctc_blocks:
                    self.interctc_modules.append(modules.InterCTCResModule(
                        dim_model=dim_model[stage_id + (1 if not transposed_block and down_block else 0)],
                        vocab_size=vocab_size
                    ))

                i += 1

    def forward(self, x, lengths):

        # Pos Embedding
        if self.pos_embedding != None:
            x = self.pos_embedding(x)

        # Dropout
        x = self.dropout(x)

        # Mask (1 or B, 1, N, N)
        if self.mask != None:
            mask = self.mask(x, lengths)
        else:
            mask = None

        # Conformer Blocks
        interctc_outputs = {}
        j = 0
        for i, block in enumerate(self.conformer_blocks):

            # Conformer Block
            x = block(x, mask=mask)

            # InterCTC Block
            if i + 1 in self.interctc_blocks:
                x, logits = self.interctc_modules[j](x)
                j += 1
                key = self.loss_prefix + "_" + str(i)
            else:
                logits = None

            # Strided Block
            if block.stride > 1:

                # Stride Mask (1 or B, 1, T // S, T // S)
                if mask is not None:
                    mask = mask[:, :, ::block.stride, ::block.stride]

                # Update Seq Lengths
                if lengths is not None:
                    lengths = torch.div(lengths - 1, block.stride, rounding_mode='floor') + 1

            if logits != None:
                interctc_outputs[key] = [logits, lengths]

        return x, lengths, interctc_outputs

class AudioEfficientConformerEncoder(nn.Module):

    def __init__(self, include_head=True, vocab_size=256, att_type="patch", interctc_blocks=[3, 6, 10, 13], num_blocks=[5, 6, 5], loss_prefix="ctc"):
        super(AudioEfficientConformerEncoder, self).__init__()

        assert att_type in ["regular", "grouped", "patch"]

        # Params
        sample_rate=16000
        n_fft=512
        win_length_ms=25
        hop_length_ms=10
        n_mels=80
        mF=2
        F=27
        mT=5
        pS=0.05
        kernel_size=15
        drop_rate=0.1
        attn_drop_rate=0.0
        max_pos_encoding=10000
        causal=False
        subsampling_filters=180
        dim_model=[180, 256, 360]
        num_heads=4

        # Audio Preprocessing (B, T) -> (B, n_mels, T // hop_length + 1)
        self.audio_preprocessing = preprocessing.AudioPreprocessing(
            sample_rate=sample_rate,
            n_fft=n_fft,
            win_length_ms=win_length_ms,
            hop_length_ms=hop_length_ms,
            n_mels=n_mels,
            normalize=False,
            mean=-5.6501,
            std=4.2280
        )

        # Spec Augment
        self.spec_augment = preprocessing.SpecAugment(
            mF=mF,
            F=F,
            mT=mT,
            pS=pS
        )

        # Unsqueeze (B, N, T) -> (B, 1, N, T)
        self.unsqueeze = layers.Unsqueeze(dim=1)

        # Stem (B, 1, N, T) -> (B, C, N', T')
        self.subsampling_module = modules.ConvNeuralNetwork(
            dim_input=1,
            dim_layers=subsampling_filters,
            kernel_size=3,
            strides=2,
            norm="BatchNorm2d",
            act_fun="Swish",
            drop_rate=0.0,
            dim=2
        )

        # Reshape (B, C, N, T) -> (B, D, T)
        self.reshape = layers.Reshape(shape=(subsampling_filters * n_mels // 2, -1), include_batch=False)

        # Transpose (B, D, T) -> (B, T, D)
        self.transpose = layers.Transpose(1, 2)

        # Linear Proj
        self.linear = layers.Linear(subsampling_filters * n_mels // 2, dim_model[0])

        # Conformer
        self.back_end = ConformerInterCTC(
            dim_model=dim_model,
            num_blocks=num_blocks,
            interctc_blocks=interctc_blocks,
            vocab_size=vocab_size,
            att_params=[
                {"class": "RelPos1dMultiHeadAttention", "params": {"num_heads": num_heads, "attn_drop_rate": attn_drop_rate, "num_pos_embeddings": max_pos_encoding, "weight_init": "default", "bias_init": "default"}},
                {"class": "RelPos1dMultiHeadAttention", "params": {"num_heads": num_heads, "attn_drop_rate": attn_drop_rate, "num_pos_embeddings": max_pos_encoding, "weight_init": "default", "bias_init": "default"}},
                {"class": "RelPos1dMultiHeadAttention", "params": {"num_heads": num_heads, "attn_drop_rate": attn_drop_rate, "num_pos_embeddings": max_pos_encoding, "weight_init": "default", "bias_init": "default"}}
            ] if att_type == "regular" else [
                {"class": "GroupedRelPosMultiHeadSelfAttention", "params": {"num_heads": num_heads, "group_size": 3, "attn_drop_rate": attn_drop_rate, "max_pos_encoding": max_pos_encoding, "causal": causal}},
                {"class": "GroupedRelPosMultiHeadSelfAttention", "params": {"num_heads": num_heads, "group_size": 1, "attn_drop_rate": attn_drop_rate, "max_pos_encoding": max_pos_encoding, "causal": causal}},
                {"class": "GroupedRelPosMultiHeadSelfAttention", "params": {"num_heads": num_heads, "group_size": 1, "attn_drop_rate": attn_drop_rate, "max_pos_encoding": max_pos_encoding, "causal": causal}}
            ] if att_type == "grouped" else [
                {"class": "RelPosPatch1dMultiHeadAttention", "params": {"num_heads": num_heads, "patch_size": 3, "attn_drop_rate": attn_drop_rate, "num_pos_embeddings": max_pos_encoding, "weight_init": "default", "bias_init": "default"}},
                {"class": "RelPos1dMultiHeadAttention", "params": {"num_heads": num_heads, "attn_drop_rate": attn_drop_rate, "num_pos_embeddings": max_pos_encoding, "weight_init": "default", "bias_init": "default"}},
                {"class": "RelPos1dMultiHeadAttention", "params": {"num_heads": num_heads, "attn_drop_rate": attn_drop_rate, "num_pos_embeddings": max_pos_encoding, "weight_init": "default", "bias_init": "default"}}
            ],
            conv_params={ "class": "Conv1d", "params": {"padding": "same", "kernel_size": kernel_size}},
            ff_ratio=4,
            drop_rate=drop_rate,
            pos_embedding=None,
            mask=attentions.Mask(),
            conv_stride=2,
            batch_norm=True,
            loss_prefix=loss_prefix
        )

        # Head
        self.head = layers.Linear(dim_model[-1], vocab_size) if include_head else nn.Identity()

    def forward(self, x, lengths):

        # Audio Preprocessing
        x, lengths = self.audio_preprocessing(x, lengths)

        # Spec Augment
        x = self.spec_augment(x, lengths)

        # Unsqueeze
        x = self.unsqueeze(x)

        # Stem
        x, lengths = self.subsampling_module(x, lengths)

        # Reshape
        x = self.reshape(x)

        # Transpose
        x = self.transpose(x)

        # Linear Proj
        x = self.linear(x)

        # Conformer
        x, lengths, interctc_outputs = self.back_end(x, lengths)

        # Head
        x = self.head(x)

        return x, lengths, interctc_outputs

class VisualEfficientConformerEncoder(nn.Module):

    def __init__(self, include_head=True, vocab_size=256, interctc_blocks=[3, 6, 9], num_blocks=[6, 6], loss_prefix="ctc"):
        super(VisualEfficientConformerEncoder, self).__init__()

        # Params
        dim_model=[256, 360]
        num_heads=4
        kernel_size=15
        drop_rate=0.1
        attn_drop_rate=0.0
        max_pos_encoding=10000

        # Stem 88 -> 44
        # MaxPool 44 -> 22
        # ResNet 22 -> 11 -> 6 -> 3
        # GlobalPool 3 -> 1
        self.front_end = nn.Sequential(
            modules.ConvNeuralNetwork(
                dim_input=1,
                dim_layers=64,
                kernel_size=(5, 7, 7),
                strides=(1, 2, 2),
                norm="BatchNorm3d",
                act_fun="ReLU",
                drop_rate=0.0,
                dim=3
            ),
            layers.MaxPool3d(kernel_size=(1, 3, 3), stride=(1, 2, 2), padding="same"),
            transforms.VideoToImages(), # (B, C, T, H, W) -> (BT, C, H, W)
            ResNet(include_stem=False, dim_output=dim_model[0], model="ResNet18")
        )

        self.expand_time = transforms.ImagesToVideos()

        # Conformer
        self.back_end = ConformerInterCTC(
            dim_model=dim_model,
            num_blocks=num_blocks,
            interctc_blocks=interctc_blocks,
            vocab_size=vocab_size,
            att_params={"class": "RelPos1dMultiHeadAttention", "params": {"num_heads": num_heads, "attn_drop_rate": attn_drop_rate, "num_pos_embeddings": max_pos_encoding, "weight_init": "default", "bias_init": "default"}},
            conv_params={"class": "Conv1d", "params": {"padding": "same", "kernel_size": kernel_size}},
            ff_ratio=4,
            drop_rate=drop_rate,
            pos_embedding=None,
            mask=attentions.Mask(),
            conv_stride=2,
            batch_norm=True,
            loss_prefix=loss_prefix
        )

        # Head
        self.head = layers.Linear(dim_model[-1], vocab_size) if include_head else nn.Identity()

    def forward(self, x, lengths):

        # Frontend
        time = x.shape[2]
        x = self.front_end(x) # (B, C, T, H, W) -> (BT, C)
        x = x.unsqueeze(dim=-1).unsqueeze(dim=-1) # (BT, C) -> (BT, C, 1, 1)
        x = self.expand_time(x, time) # (BT, C, 1, 1) -> (B, C, T, 1, 1)
        x = x.squeeze(dim=-1).squeeze(dim=-1).transpose(1, 2) # (B, C, T, 1, 1) -> (B, T, C)

        # Backend
        x, lengths, interctc_outputs = self.back_end(x, lengths)

        # Head
        x = self.head(x)

        return x, lengths, interctc_outputs

class AudioVisualEfficientConformerEncoder(nn.Module):

    def __init__(self, include_head=True, vocab_size=256, v_interctc_blocks=[3, 6], a_interctc_blocks=[8, 11], f_interctc_blocks=[2]):
        super(AudioVisualEfficientConformerEncoder, self).__init__()

        # Params
        dim_model = 360
        num_blocks = 5
        num_heads = 4
        drop_rate = 0.1
        attn_drop_rate = 0.0
        max_pos_encoding = 10000
        kernel_size = 15
        v_num_blocks = [6, 1]
        a_num_blocks = [5, 6, 1]

        # Visual Encoder
        self.video_encoder = VisualEfficientConformerEncoder(include_head=False, vocab_size=vocab_size, interctc_blocks=v_interctc_blocks, num_blocks=v_num_blocks, loss_prefix="v_ctc")

        # Audio Encoder
        self.audio_encoder = AudioEfficientConformerEncoder(include_head=False, vocab_size=vocab_size, interctc_blocks=a_interctc_blocks, num_blocks=a_num_blocks, loss_prefix="a_ctc")

        # Fusion Module
        self.fusion_module = modules.FusionModule(a_dim_model=dim_model, v_dim_model=dim_model, f_dim_model=dim_model)

        # Audio-visual Encoder
        self.audio_visual_encoder = ConformerInterCTC(
            dim_model=dim_model,
            num_blocks=num_blocks,
            interctc_blocks=f_interctc_blocks,
            vocab_size=vocab_size,
            att_params={"class": "RelPos1dMultiHeadAttention", "params": {"num_heads": num_heads, "attn_drop_rate": attn_drop_rate, "num_pos_embeddings": max_pos_encoding, "weight_init": "default", "bias_init": "default"}},
            conv_params={"class": "Conv1d", "params": {"padding": "same", "kernel_size": kernel_size}},
            ff_ratio=4,
            drop_rate=drop_rate,
            pos_embedding=None,
            mask=attentions.Mask(),
            conv_stride=2,
            batch_norm=True,
            loss_prefix="f_ctc"
        )

        # Head
        self.head = layers.Linear(dim_model, vocab_size) if include_head else nn.Identity()

    def forward(self, video, video_len, audio, audio_len):

        # Visual Encoder
        video, video_len, video_interctc_outputs = self.video_encoder(video, video_len)

        # Audio Encoder
        audio, audio_len, audio_interctc_outputs = self.audio_encoder(audio, audio_len)

        # Fusion Module
        x = self.fusion_module(audio, video)
        lengths = audio_len

        # Audio-visual Encoder
        x, lengths, interctc_outputs = self.audio_visual_encoder(x, lengths)
        interctc_outputs.update(video_interctc_outputs)
        interctc_outputs.update(audio_interctc_outputs)

        # Head
        x = self.head(x)

        return x, lengths, interctc_outputs


"""myblog URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.11/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url
from django.contrib import admin
from codeserver import views as codeserver_views
from runcode import views as runcode_views

if __name__ == '__main__':
    main()
urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url(r'^code/(\S{8})$',codeserver_views.codeserverIndex,name="codeserverIndex"),
    url(r'^codeup/',codeserver_views.codeup),
    url(r'^codeupdate/',codeserver_views.codeupdate),
    url(r'^runcode/',runcode_views.runcode),
    url(r'^$',runcode_views.index)
]


from pytest import fixture, mark

from flaticon import FlatIconClient

pytestmark = mark.asyncio

@fixture
def client() -> FlatIconClient:
    return FlatIconClient()

async def test_download(client: FlatIconClient):
    images = await client.search_for_images("cat", count=2)
    image_paths = [await client.download_image(image) for image in images]
    assert image_paths


#!/usr/bin/python
from __future__ import division
from __future__ import print_function
"""
This file serves as an example of how to
a) select a problem to be solved
b) select a network type
c) train the network to minimize recovery MSE

"""
import numpy as np
import os

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # BE QUIET!!!!
os.environ['KMP_DUPLICATE_LIB_OK']='True'
import tensorflow as tf

np.random.seed(1) # numpy is good about making repeatable output
tf.set_random_seed(1) # on the other hand, this is basically useless (see issue 9171)

# import our problems, networks and training modules
from tools import problems,networks,train

# Create the basic problem structure.
prob = problems.bernoulli_gaussian_trial(kappa=None,M=250,N=500,L=1000,pnz=.1,SNR=40) #a Bernoulli-Gaussian x, noisily observed through a random matrix
#prob = problems.random_access_problem(2) # 1 or 2 for compressive random access or massive MIMO
# print('Problem created ...')
# print('A is:')
# print(prob.A)

filename = 'LAMP_bg_giid.npz'

other = {}
B_t = []
U_t = []
S_t = []
V_t = []
theta_t = []

try:
    filecontent = np.load(filename).items()
    for k, d in filecontent:
        if k.startswith('B_'):
            B_t.append(d)
            print('restoring ' + k + ' is:' + str(d))
        elif k.startswith('theta_'):
            theta_t.append(d)
except IOError:
    pass

A = prob.A
M,N = A.shape
U_A, S_A, Vh_A = np.linalg.svd(A.transpose())

for i in range(len(B_t)):
    print(str(i))
    U, S, Vh = np.linalg.svd( B_t[i])
    U_t.append(U)
    S_t.append(S)
    V_t.append(Vh)

dot_prod = U_A.transpose().dot(U_A)

test = np.amax(dot_prod, axis=0)

stop = 1


"""
Runtime: 32 ms, faster than 99.63% of Python3 online submissions for Output Contest Matches.
Memory Usage: 13.6 MB, less than 27.96% of Python3 online submissions for Output Contest Matches.
"""

class Solution:
    def findContestMatch(self, n: int) -> str:
        def helper(matches):
            i , j = 0 , len(matches) - 1
            res = []
            while i < j:
                line = '(' + matches[i] + ',' + matches[j] + ')'
                res.append(line)
                i += 1
                j -= 1
            return res
        matches = [str(x) for x in range(1 , n + 1)]
        while len(matches) != 1:
            matches = helper(matches)
        return matches[0]


# # notimailer - Notification content generator.
# All exceptions thrown here will be ValueError exceptions.

#from smtplib import SMTP
import smtplib
import jinja2

# TODO: Support "profiles" for various use cases
# Allow profile to (further) dictate what kinda things (keys) are expected to appear in input data

# Construct notification mailer
class notimailer(object):
  smtphost = None
  def __init__(self, conf, **kwargs):
    if not conf: raise ValueError("No notimailer config");
    if not conf["sysenv"]: raise ValueError("Environment config missing !");
    se = conf["sysenv"]
    notimailer.smtphost = se.get("smtphost")
    if not notimailer.smtphost: raise ValueError("SMTP Host missing !");
    self.tmpl  = conf["tmpl"]
    self.stmpl = conf["subjtmpl"]
    self.smtpfrom = se.get("smtpfrom")
    ####################
    #def connect():
    self.server = smtplib.SMTP()
    self.server.connect(host=notimailer.smtphost, port=25)
    if kwargs.get("debug"): self.server.set_debuglevel(1)
    #helo = self.server.helo(name='Bill') # Returns tuple
    #print("Helo:" + str(helo) + ", in-obj:" + self.server.helo_resp + ".")
    # load templates
    print("Subject Template: " + self.stmpl);
    self.bodytmpl = notimailer.loadtemplate(self.tmpl)
    self.subjtmpl = notimailer.loadtemplate(self.stmpl, iscont=True)
    #print("Templates OK");
    if not self.bodytmpl: raise ValueError("No Body Template");
    if not self.subjtmpl: raise ValueError("No Subject Template");
  # Construct template by loading it from file or using passed value as template string.
  # By passing iscont=True in kwargs, fname parameter is expected to be the template content
  # not a filename.
  @staticmethod
  def loadtemplate(fname, **kwargs):
    tstr = ""
    if kwargs.get('iscont'):
      tstr = fname
    else:
      tpath = kwargs.get("basepath") or "./"
      tstr = open(tpath + fname, "r").read(); # "gerrit_open_notify.j2"

    if kwargs.get("debug") > 2: print("Template:" + tstr); exit(0);
    print("Template Content NOW:" + tstr);
    #NOTUSED:template = jinja2.Template(tstr)
    # NOTE: Jinja2 seems to mandate a "\n" in tstr (not a single liner snippet) !
    # OLD Complex "Environemnt" way of creating template
    #env = jinja2.Environment(tstr)
    #template = env.from_string(tstr)
    template = jinja2.Template(tstr)
    if kwargs.get("debug") > 2: print("Env: " + str(env) + ", template:" + str(template) + ".");
    return template

  # Email people related to long pending changes.
  # Params:
  # - changes - Gerrit changes structures with added-on  manager info
  # - accts_idx - Fast lookup index for full user accounts info
  # TODO: Pass pre-prepared templates and email config.
  # kwargs:
  # - debug - Verbose output
  # - noemail - No actual emailing to the people to be notified
  # - adminto - Admin email addresses to send a summarizing (with all messages concatenated) email to
  # - adminsample - Send only single email as "actual sample" to adminlist (Looking exactly like real recipient would have it)
  # - tdictcb - Template dictionary callback
  # Entries in param changes must have keys:
  # - wtid - Work task id label
  # - subject - Work task reabable name / description
  def notify(self, changes, accts_idx, **kwargs):
    if not self.smtpfrom: raise ValueError("No From address for notification");
    #smtpfrom = "ccxswbuild@broadcom.com" # smtpfrom = this.smtpfrom
    emailcfg = {"from": self.smtpfrom, "to": None}
    server = self.server # SMTP() # Server may be passed already here.
    #smtpconn =

    ############ Templating ##############
    #template = notimailer.loadtemplate(self.tmpl);
    #if not template: print("No Template");exit(1);
    #exit(0)

    i = 0
    collout = "" # Cumulated / Collective output (e.g. to admin)
    doemail = not kwargs.get("noemail")
    adminto = kwargs.get("adminto")
    tdictcb = kwargs.get("tdictcb") or None
    totnote = ""
    subject = ""
    if adminto and (len(adminto) > 0): emailcfg["to"] = adminto

    #print("doemail: " + str(doemail)); exit(1);
    for ch in changes:
      #
      chid = ch["wtid"]
      #OLD:uid = ch["owner"]["_account_id"] # NEW: "userkey"
      uid = ch["userkey"]
      # u - User. Has full name in "name"
      u = accts_idx.get(str(uid), None)
      if not u: print("No User " +str(uid) + " in index (skip...)."); continue
      if not u["name"]: print("Warning: User " +str(uid) + " has no 'name' for email full name display."); continue
      if kwargs.get("debug") > 1: print(json.dumps(u) + "\n");
      # Template params. Deep clone u to base params on ?
      p = None
      if tdictcb: p = tdictcb(ch, u)
      else: p = { "name": u["name"], "wtid": chid, "subject": ch["subject"], "age_d": ch["age_d"] }
      if not u: print("No user " + str(uid) + " in (accts_idx) Index !"); continue
      if not u.get("email"): print("No user " + str(uid) + " email info in (accts_idx) Index !"); continue

      # DONOTUSE: recipients = [ u["email"] ] # Primary
      recipients = [ u["name"] + " <" + u["email"] + ">" ]
      if u.get("manager") and u["manager"].get("email"):
        mgr = u["manager"]
        if not mgr["name"]: print("Warning: User's manager " +str(mgr["username"]) + " has no 'name' for email full name display."); continue
        recipients.append(mgr["name"] + " <" + mgr["email"] + ">") # Add manager (Old/plain: mgr["email"])
      out = self.bodytmpl.render(**p)
      print(str(i) + ") Send Work Task Notif.:" + str(chid) + " To:" + ', '.join(recipients) + ".")
      #print("OUT:" + out);
      collout += out # Add to collective summary (Sent to admin)
      # Note: smtplib wants recipients redundantly in headers (!)
      sstr = self.subjtmpl.render(**p)
      out = "To: " + ", ".join(recipients) + "\nSubject: "+ sstr +"\n\n" + out # Add subject
      if kwargs.get("adminsample"): collout = out; break
      if doemail: server.sendmail(emailcfg["from"], recipients, out)
      else: print(out) # Merely debug
      i += 1
    # Summarizing Admin Email (TODO: Template)
    # SUMMARIZING TEST OUTPUT FOLLOWS.
    if not kwargs.get("adminsample") and emailcfg["to"]:
      totnote = "THESE NOTIFICATIONS ARE TO BE SENT OUT INDIVIDUALLY.\n" + str(i) + " Notifications going out on Work Tasks:\n"
      subject = "To: "+", ".join(emailcfg["to"])+"\nSubject: Work Task notifications ("+str(i)+")\n\n"
    # Note: headers MUST be bundeled to msg (!) with "\n\n" delimiting headers and body

    # Admin email
    # .. or to some DL, e.g. (?)
    #emailcfg["to"] = "\"CCXSW-DEVOPS-LIST,PDL\" <ccxsw-devops-list.pdl@broadcom.com>"
    if emailcfg["to"]:
      print("Sending out Summarizing Admin email: "+" ".join(emailcfg["to"])+"...");
      server.sendmail(emailcfg["from"], emailcfg["to"], subject + totnote + collout)
    server.quit()
    return i


import tensorflow as tf

'''
MNIST example using fully connected layers
'''

mnist = tf.keras.datasets.mnist
(train_images, train_labels), (test_images, test_labels) = mnist.load_data()

INPUT_SHAPE = train_images.shape[1:]
BATCH_SIZE = 32
EPOCHS = 5

# preprocessing
train_images = train_images / 255.0
test_images = test_images / 255.0

def simple_model():
  input = tf.keras.layers.Input(shape=(INPUT_SHAPE+(1,))) #for gray scale image
  net = tf.keras.layers.Flatten()(input)  #flatten data
  net = tf.keras.layers.Dense(200, activation='relu')(net)
  net = tf.keras.layers.Dense(200, activation='relu')(net)

  output = tf.keras.layers.Dense(10, activation='sigmoid')(net)

  return tf.keras.Model(input,output)

model = simple_model()
model.summary() #show model

model.compile(optimizer='adam', loss= 'sparse_categorical_crossentropy',metrics=['accuracy'])
model.fit(train_images, train_labels, batch_size=BATCH_SIZE, epochs=EPOCHS
          , validation_data=(test_images, test_labels)
          )


from sklearn.model_selection import train_test_split #这里是引用了交叉验证，分成训练集和测试集
from sklearn.linear_model import LinearRegression  #线性回归
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import statsmodels.api as sm
import scipy.stats as stats

###读取数据
pd_data=pd.read_excel('C:\\Users\\pyca\\Desktop\\毕业设计\\job_info_num.xlsx')
X = pd_data.loc[:, ('工作经验', '公司规模', '城市招聘发布数', '行业招聘发布数'
                    , 'excel', 'python',  'hadoop', 'BI')]
y = pd_data.loc[:, '薪资']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=5)
print('X_train.shape={}\n y_train.shape ={}\n X_test.shape={}\n,  y_test.shape={}'.format(X_train.shape, y_train.shape,
                                                                                X_test.shape, y_test.shape))
linreg = LinearRegression()
model = linreg.fit(X_train, y_train)
feature_cols = ['工作经验', '学历', '公司规模', '城市招聘发布数', '行业招聘发布数'
    , 'excel', 'python', 'hadoop', 'BI', '薪资']
B = list(zip(feature_cols, linreg.coef_))
y_pred = linreg.predict(X_test)

L = []
for i in range(len(y_pred)):
    a = y_pred[i] - y_test.values[i]
    L.append(a)
print(len(L))

# name=['residual']
# test=pd.DataFrame(columns=name,data=L)#数据有一列列，列名为residual
# test.to_csv('C:/Users/pyca/Desktop/毕业设计/4、多元线性回归/Multiple-Linear-Regression-master/Multiple-Linear-Regression-master/code/\
# residualtest.csv',encoding='gbk')

残差检验-计算残差并存入excel.py

残差检验-计算残差并存入excel.py


import itertools
import collections

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from utils import *

def load_encoding_results():
    cache = "../../braincode/.cache"
    table = collections.defaultdict(list)
    for region, model in itertools.product(BRAIN.items(), MODELS.items()):
        prefix = "nlea"
        score = np.tanh(
            np.load(
                f"{cache}/scores/{prefix}/score_{region[0]}_{model[0]}_FisherCorr.npy"
            )
        )
        subjects = np.tanh(
            np.load(
                f"{cache}/scores/{prefix}/subjects_{region[0]}_{model[0]}_FisherCorr.npy"
            )
        )
        table["Analysis"].append(prefix)
        table["Region"].append(region[1])
        table["Model"].append(model[1])
        table["Score"].append(score)
        table["Subjects"].append(subjects)
    return pd.DataFrame(table)

def plot_model_encoding_by_network(data):
    for i, region in enumerate(data["Region"].unique()):
        ax = plt.subplot(3, 2, i + 1)
        samples = data[data["Region"] == region]
        baseline = samples[samples["Model"] == "Token Projection"].Score.values
        samples_nl = samples[samples["Model"].str.contains("NL")]
        scores_nl = samples_nl.Score.values
        err_nl = [arr.std() / np.sqrt(arr.size) for arr in samples_nl["Subjects"]]
        samples_py = samples[samples["Model"].str.contains("Py")]
        scores_py = samples_py.Score.values
        err_py = [arr.std() / np.sqrt(arr.size) for arr in samples_py["Subjects"]]
        n_params = [350e6, 2e9, 6e9, 16e9]
        plt.axhline(baseline, color="black", linestyle="--", label="Token Projection")
        plt.errorbar(
            n_params, scores_nl, err_nl, color="forestgreen", capsize=2, label="NL"
        )
        plt.errorbar(
            n_params, scores_py, err_py, color="darkgreen", capsize=2, label="Python"
        )
        plt.title(region, fontweight="bold")
        plt.xlabel("# of Model Parameters", fontweight="bold") if i == 2 else None
        plt.ylabel("Encoding Score (Pearson R)", fontweight="bold") if i == 2 else None
        plt.xscale("log")
        plt.ylim([-0.1, 0.3])
        for spine in ["left", "right", "top", "bottom"]:
            ax.spines[spine].set_visible(False)
        ax.yaxis.set_ticks_position("left")
        ax.xaxis.set_ticks_position("bottom")
    ax = plt.subplot(3, 2, 5)
    plt.axhline(baseline, color="black", linestyle="--", label="Token Projection")
    plt.errorbar(
        n_params, scores_nl, err_nl, color="forestgreen", capsize=2, label="NL"
    )
    plt.errorbar(
        n_params, scores_py, err_py, color="darkgreen", capsize=2, label="Python"
    )
    h, l = ax.get_legend_handles_labels()
    ax.clear()
    ax.legend(h, l, loc="center left")
    ax.axis("off")
    plt.gcf().set_size_inches(6, 6)
    plt.tight_layout()
    plt.savefig(f"fig_enc_brain_models_byregion.png", bbox_inches="tight", dpi=600)
    plt.close()

def plot_model_encoding_by_corpus(data):
    for i, corpus in enumerate(["NL", "Python"]):
        ax = plt.subplot(2, 2, i + 1)
        samples = data[data["Model"].str.contains(corpus)]
        samples_md_lh = samples[samples["Region"] == "MD LH"]
        scores_md_lh = samples_md_lh.Score.values
        err_md_lh = [arr.std() / np.sqrt(arr.size) for arr in samples_md_lh.Subjects]
        samples_md_rh = samples[samples["Region"] == "MD RH"]
        scores_md_rh = samples_md_rh.Score.values
        err_md_rh = [arr.std() / np.sqrt(arr.size) for arr in samples_md_rh.Subjects]
        samples_lang_lh = samples[samples["Region"] == "Lang LH"]
        scores_lang_lh = samples_lang_lh.Score.values
        err_lang_lh = [
            arr.std() / np.sqrt(arr.size) for arr in samples_lang_lh.Subjects
        ]
        samples_lang_rh = samples[samples["Region"] == "Lang RH"]
        scores_lang_rh = samples_lang_rh.Score.values
        err_lang_rh = [
            arr.std() / np.sqrt(arr.size) for arr in samples_lang_rh.Subjects
        ]
        n_params = [350e6, 2e9, 6e9, 16e9]
        plt.errorbar(
            n_params,
            scores_md_lh,
            err_md_lh,
            color="navy",
            capsize=2,
            label="MD LH",
        )
        plt.errorbar(
            n_params,
            scores_md_rh,
            err_md_rh,
            color="royalblue",
            capsize=2,
            label="MD RH",
        )
        plt.errorbar(
            n_params,
            scores_lang_lh,
            err_lang_lh,
            color="maroon",
            capsize=2,
            label="Lang LH",
        )
        plt.errorbar(
            n_params,
            scores_lang_rh,
            err_lang_rh,
            color="indianred",
            capsize=2,
            label="Lang RH",
        )
        plt.title(corpus, fontweight="bold")
        plt.xlabel("# of Model Parameters", fontweight="bold")
        plt.ylabel("Encoding Score (Pearson R)", fontweight="bold")
        plt.xscale("log")
        plt.ylim([-0.1, 0.3])
        for spine in ["left", "right", "top", "bottom"]:
            ax.spines[spine].set_visible(False)
        ax.yaxis.set_ticks_position("left")
        ax.xaxis.set_ticks_position("bottom")
    ax = plt.subplot(2, 2, 3)
    plt.errorbar(
        n_params,
        scores_md_lh,
        err_md_lh,
        color="navy",
        capsize=2,
        label="MD LH",
    )
    plt.errorbar(
        n_params,
        scores_md_rh,
        err_md_rh,
        color="royalblue",
        capsize=2,
        label="MD RH",
    )
    plt.errorbar(
        n_params,
        scores_lang_lh,
        err_lang_lh,
        color="maroon",
        capsize=2,
        label="Lang LH",
    )
    plt.errorbar(
        n_params,
        scores_lang_rh,
        err_lang_rh,
        color="indianred",
        capsize=2,
        label="Lang RH",
    )
    h, l = ax.get_legend_handles_labels()
    ax.clear()
    ax.legend(h, l, loc="center left")
    ax.axis("off")
    plt.gcf().set_size_inches(6, 4)
    plt.tight_layout()
    plt.savefig(f"fig_enc_brain_models_bycorpus.png", bbox_inches="tight", dpi=600)
    plt.close()

def plot_model_encoding_correlations(data):
    for i, (region_a, region_b) in enumerate(
        itertools.combinations(data.Region.unique(), 2)
    ):
        samples_a = np.concatenate(data[data["Region"] == region_a]["Subjects"].values)
        samples_b = np.concatenate(data[data["Region"] == region_b]["Subjects"].values)
        ax = plt.subplot(2, 3, i + 1)
        plt.scatter(samples_a, samples_b, color="black", s=1)
        scale = 0.8
        plt.xlabel(region_a, fontweight="bold")
        plt.ylabel(region_b, fontweight="bold")
        plt.xlim([-scale, scale])
        plt.ylim([-scale, scale])
        for spine in ["left", "right", "top", "bottom"]:
            ax.spines[spine].set_visible(False)
        ax.yaxis.set_ticks_position("left")
        ax.xaxis.set_ticks_position("bottom")
    plt.tight_layout()
    plt.savefig(f"fig_enc_brain_models_regioncorrs.png", bbox_inches="tight", dpi=600)
    plt.close()

def main():
    data = load_encoding_results()
    plot_model_encoding_by_network(data)
    plot_model_encoding_by_corpus(data)
    plot_model_encoding_correlations(data)

if __name__ == "__main__":
    main()


#!/usr/bin/env python3
import sys
import argparse
import subprocess
import os.path
import random
from Bio import SeqIO

def coverageCalculator(readdict, assemblylength):
    #print(readdict)
    reads = [len(sequence) for id, sequence in readdict.items()]
    #print(reads)
    return sum(reads) / assemblylength

def readDownSampler(readfile, assemblylength, coveragecutoff, lengthcutoff):
    #Creating a dictionary containing ids and sequences of the reads
    readdict = {(read.id):(read.seq) for read in (SeqIO.parse(readfile, "fastq")) if len(read.seq) > lengthcutoff}

    coverage = coverageCalculator(readdict, assemblylength)

    print("Downsampling", readfile)
    print("Assumed fragment length:", assemblylength)
    print("Start coverage:", coverage)
    while coverage > coveragecutoff:
        for _ in range(int((coverage / coveragecutoff))):
            readdict.pop(random.choice(list(readdict.keys())))
        coverage = coverageCalculator(readdict, assemblylength)
    print("Final coverage:", coverage)
    return readdict

def writeDownsampledReads(readdict, outputfile,fastqfile):
    outfile = open(outputfile, 'w')

    for read in SeqIO.parse(fastqfile, "fastq"):
        if read.id in readdict:
            outfile.write(read.format('fastq'))

def findFragmentLength(fragmentlengthfile, allelename):
    allele = allelename.split('_')[0]
    with open(fragmentlengthfile,'r') as file:
        for line in file:
            if line.startswith(allele):
                file = open(f'{line.split()[1]}.{allelename}','w')
                file.close()
                return int(line.split()[1])

class CustomParser(argparse.ArgumentParser):
    def error(self, message):
        sys.stderr.write('error: %s\n' % message)
        self.print_help()
        sys.exit(2)

def arguments():
    parser = CustomParser(description='Genotyper for HLA.')
    parser.add_argument('--readfile', type = str)
    parser.add_argument('--outputfile', type = str)
    parser.add_argument('--coveragecutoff', type = int, default = 100)
    parser.add_argument('--fragmentlength', type = str, default = None)
    parser.add_argument('--allele', type = str, default = None)
    parser.add_argument('--readsizecutoff', type = int, default = 2000)

    return parser.parse_args()

def main():
    args = arguments()
    if args.allele is not None:
        assemblylength = findFragmentLength(args.fragmentlength, args.allele)
    else:
        assemblylength = 3200

    readdict = readDownSampler(args.readfile, assemblylength, args.coveragecutoff, args.readsizecutoff)
    writeDownsampledReads(readdict, args.outputfile, args.readfile)

main()


# This file is part of JST.
#
# JST is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# JST is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with JST.  If not, see <http://www.gnu.org/licenses/>.

import math

from symbol_table.scope import Scope
from symbol_table.symbol import Symbol, VariableSymbol

# This is where MARS starts the global data memory
# We may need to watch out for memory things like .asciiz grab
# So perhaps we should start somewhere else?
# (Shubham) If we allocate everything manually everything, we can ignore the .asciiz grab
MIPS_DATA_MEMORY_BASE = 0x10010000

class SymbolTable(object):

    # Initializes the symbol table
    def __init__(self):
        self.table = []

        self.next_data_memory_location = MIPS_DATA_MEMORY_BASE
        # Increments by the type size
        self.next_activation_frame_offset = 0

    # Pushes a scope onto the table.
    #
    # @param scope Scope to push. Default pushes an empty Scope.
    def push(self, scope=None):
        if scope is None:
            scope = Scope()
        if type(scope) is Scope:
            self.table.append(scope)
        else:
            raise TypeError("'scope' is not an instance of Scope.")

    # Pops the top-most Scope from the table and returns it.
    def pop(self):
        # The second scope is first scope of a function
        # if len(self.table) == 2:
        #     self.next_activation_frame_offset = 0

        return self.table.pop()

    # Inserts a symbol into the top-most Scope.
    #
    # @param symbol Symbol to insert.
    #
    # @return Tuple of the result and a list of shadowed Symbols or re-declared Symbol.
    def insert(self, symbol):
        if not isinstance(symbol, Symbol):
            raise TypeError("parameter 'symbol' is not an instance of Symbol. Actual type: {}".format(type(symbol)))
        elif not self.table:
            raise Exception('Table has no scopes available to insert into. Offending symbol: {}'.format(symbol))

        if isinstance(symbol, VariableSymbol):
            if symbol.is_array:
                if symbol.is_parameter:
                    # 1 word for the base address
                    # 1 word for total array size in bytes
                    # x words for the bound on each dimension
                    required_byte_size = 4 * (len(symbol.array_dims) + 2)
                else:
                    required_byte_size = math.ceil((symbol.size_in_bytes() * symbol.array_size) / 4) * 4
            else:
                required_byte_size = math.ceil(symbol.size_in_bytes() / 4) * 4

            if len(self.table) == 1:
                # symbol.global_memory_location = self.next_data_memory_location
                # self.next_data_memory_location += required_byte_size

                symbol.global_memory_location = symbol.identifier

            else:
                symbol.activation_frame_offset = self.next_activation_frame_offset
                self.next_activation_frame_offset += required_byte_size

        shadowed_symbols = []
        for scope in self.table:
            result = scope.find(symbol.identifier)
            if result is not None:
                if scope is not self.table[-1]:
                    shadowed_symbols.append(result)
                else:
                    return Scope.INSERT_REDECL, [result]

        self.table[-1].insert(symbol)

        if len(shadowed_symbols) is 0:
            return Scope.INSERT_SUCCESS, []
        else:
            return Scope.INSERT_SHADOWED, shadowed_symbols

    # Finds a symbol in the table by searching the top-most Scope to the
    # bottom-most Scope.
    #
    # @param name String identifier for the Symbol to find.
    #
    # @return Tuple of the Symbol and the Scope index it was found in.
    def find(self, name):
        for index, scope in enumerate(reversed(self.table)):
            result = scope.find(name)
            if result is not None:
                scope_level = (len(self.table) - (index + 1))
                return result, scope_level
        return None, None

    # Finds a symbol in the table by searching only the top-most Scope.
    #
    # @param name String identifier for the Symbol to find.
    #
    # @return 'None' or the Symbol.
    def find_in_top(self, name):
        return self.table[-1].find(name)

    # Finds a type in the table by searching the top-most Scope to the
    # bottom-most Scope.
    #
    # @param name String identifier for the type to find.
    #
    # @return Tuple of the type and the Scope index it was found in.
    def find_type(self, identifier):
        pass

    # Size of the table.
    #
    # @return Number of scopes in the table.
    def size(self):
        return len(self.table)

    # Clones the current SymbolTable and returns a deep copy.
    #
    # @return Copy of the SymbolTable.
    def clone(self):
        result = SymbolTable()
        result.table = []
        for scope in self.table:
            result.table.append(scope.clone())
        return result

    # @return String representation of the current SymbolTable.
    def __repr__(self):
        scopes = []
        for index, scope in enumerate(self.table):
            scopes.append('Scope #' + repr(index) + '\n' + repr(scope))
        return '\n'.join(scopes)


from threading import Thread
import time

test = True
first = 1
def func1():
    global first
    global test
    if first==1:
        print("1")
    else:
        func3()
    first = first + 1
    test = False
    print("test1", test)
def func2():
    print("2")

def func3():
    print("3")

num = 1
while test:
    print("test2", test)
    if num <6:
        Thread(target = func1).start()
        Thread(target = func2).start()
        time.sleep(2)
    else:
        break
    print("num",num)
    num = num +1


# %%
import cv2
import numpy as np
import dlib
from math import hypot
from matplotlib import pyplot as plt

# %%
cap = cv2.VideoCapture(0)

detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")

font = cv2.FONT_HERSHEY_PLAIN

max_ratio = 5
min_ratio = 4

x_cor = []
y_cor = []
y_cor_2 = []
i = 0
start_time = 0
blink_num = 0
half_blink_num = 0
plt.ion()

def midpont(p1, p2):
    return int((p1.x + p2.x)/2), int((p1.y + p2.y)/2)

def get_blinking_ratio(eye_points, facial_landmarks):
    left_point = (facial_landmarks.part(eye_points[0]).x, facial_landmarks.part(eye_points[0]).y)
    right_point = (facial_landmarks.part(eye_points[3]).x, facial_landmarks.part(eye_points[3]).y)

    center_top = midpont(facial_landmarks.part(eye_points[1]), facial_landmarks.part(eye_points[2]))
    center_bottom = midpont(facial_landmarks.part(eye_points[4]), facial_landmarks.part(eye_points[5]))

    hor_line = cv2.line(frame, left_point, right_point, (0, 255, 0), 2)
    ver_line = cv2.line(frame, center_top, center_bottom, (0, 255, 0), 2)

    hor_line_length = hypot((left_point[0] - right_point[0]), (left_point[1] - right_point[1]))
    ver_line_length = hypot((center_top[0] - center_bottom[0]), (center_top[1] - center_bottom[1]))

    length = int(abs(left_point[0]-right_point[0]))

    eye_area = cv2.rectangle(frame, (left_point[0], int(left_point[1]-hor_line_length/3)),
                             (right_point[0], int(right_point[1]+hor_line_length/3)), (0, 255, 0), 2)

    ratio = hor_line_length / ver_line_length
    return ratio

while True:
    _, frame = cap.read()
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    faces = detector(gray)
    for face in faces:
        start_time += 1

        x, y = face.left(), face.top()
        x1, y1 = face.right(), face.bottom()
        cv2.rectangle(frame, (x, y), (x1, y1), (0, 255, 0), 2)
        landmarks = predictor(gray, face)

        # x1_eye = landmarks.part(36).x
        # x2_eye = landmarks.part(39).x
        #
        # y = landmarks.part(36).y
        # cv2.circle(frame, (x, y), 3, (0, 0, 255), 2)

        # Get both eyes ratio
        left_eye_ratio = get_blinking_ratio([36, 37, 38, 39, 40, 41], landmarks)
        right_eye_ratio = get_blinking_ratio([42, 43, 44, 45, 46, 47], landmarks)

        # the average ratio
        blinking_ratio = (left_eye_ratio+right_eye_ratio)/2

        if 50 < start_time < 200:
            text0 = 'please look at it'
            cv2.putText(frame, text0, (100, 80), font, 3, (0, 0, 255), 3)
            hor_line = cv2.line(frame, (700, 150), (800, 150), (0, 0, 255), 2)
            ver_line = cv2.line(frame, (750, 100), (750, 200), (0, 0, 255), 2)

            # Find max and min programe
            if blinking_ratio > max_ratio:
                max_ratio = blinking_ratio

            if blinking_ratio < min_ratio:
                min_ratio = blinking_ratio

            text1 = 'max ratio is ' + str(max_ratio)[:5]
            text2 = 'min ratio is ' + str(min_ratio)[:5]
            cv2.putText(frame, text1, (50, 200), font, 1, (0, 255, 0), 3)
            cv2.putText(frame, text2, (50, 240), font, 1, (0, 255, 0), 3)

        if start_time > 220:
            i += 1
            x_cor.append(i)
            y_cor.append(blinking_ratio)
            y_cor_2.append(right_eye_ratio)

            plt.clf()

            plt.plot(x_cor, y_cor)
            # plt.plot(x_cor, y_cor_2)

            plt.pause(0.001)
            plt.ioff()  # close poloting window

            text1 = 'max ratio is ' + str(max_ratio)[:5]
            text2 = 'min ratio is ' + str(min_ratio)[:5]
            text3 = 'blinking num ' + str(blink_num)[:5]
            text4 = 'half blinking num ' + str(half_blink_num)[:5]

            if blinking_ratio > max_ratio*0.7:
                blink_num += 1

            if max_ratio*0.5 < blinking_ratio < max_ratio*0.7:
                half_blink_num +=1

            cv2.putText(frame, text1, (50, 150), font, 2, (0, 255, 0), 4)
            cv2.putText(frame, text2, (50, 200), font, 2, (0, 255, 0), 4)
            cv2.putText(frame, text3, (50, 250), font, 2, (0, 255, 0), 4)
            cv2.putText(frame, text4, (50, 300), font, 2, (0, 255, 0), 4)
            cv2.putText(frame, str(blinking_ratio), (50, 350), font, 2, (0, 255, 0), 4)

    cv2.imshow('Frame', frame)

    key = cv2.waitKey(20)
    if key == 27:
        break

cap.release()
cv2.destroyAllWindows()


"""JSON service for HUC12 data"""
import datetime

import pandas as pd

# needed for Decimal formatting to work
import simplejson as json
from paste.request import parse_formvars
from pyiem.dep import RAMPS
from pyiem.util import get_sqlalchemy_conn, utc
from pymemcache.client import Client

def do(ts, ts2):
    """Do work"""
    with get_sqlalchemy_conn("idep") as conn:
        df = pd.read_sql(
            """
        with data as (
            SELECT huc_12,
            sum(coalesce(avg_loss, 0)) * 4.463 as avg_loss,
            sum(coalesce(avg_delivery, 0)) * 4.463 as avg_delivery,
            sum(coalesce(qc_precip, 0)) / 25.4 as qc_precip,
            sum(coalesce(avg_runoff, 0)) / 25.4 as avg_runoff
            from results_by_huc12 WHERE valid >= %s and valid <= %s
            and scenario = 0 GROUP by huc_12)

        SELECT h.huc_12,
        coalesce(round(d.avg_loss::numeric, 2), 0) as avg_loss,
        coalesce(round(d.qc_precip::numeric, 2), 0) as qc_precip,
        coalesce(round(d.avg_delivery::numeric, 2), 0) as avg_delivery,
        coalesce(round(d.avg_runoff::numeric, 2), 0) as avg_runoff
        from huc12 h LEFT JOIN data d ON (h.huc_12 = d.huc_12) WHERE
        h.scenario = 0
        """,
            conn,
            params=(ts, ts2),
            index_col=None,
        )
    res = {
        "data": df.to_dict(orient="records"),
        "date": ts.strftime("%Y-%m-%d"),
        "date2": None if ts2 is None else ts2.strftime("%Y-%m-%d"),
        "generation_time": utc().strftime("%Y-%m-%dT%H:%M:%SZ"),
        "count": len(df.index),
    }
    myramp = RAMPS["english"][0]
    if ts2 is not None:
        days = (ts2 - ts).days
        myramp = RAMPS["english"][1]
        if days > 31:
            myramp = RAMPS["english"][2]

    res["ramps"] = dict(
        avg_loss=myramp,
        qc_precip=myramp,
        avg_delivery=myramp,
        avg_runoff=myramp,
    )
    res["max_values"] = dict(
        avg_loss=df["avg_loss"].max(),
        qc_precip=df["qc_precip"].max(),
        avg_delivery=df["avg_delivery"].max(),
        avg_runoff=df["avg_runoff"].max(),
    )
    return json.dumps(res)

def application(environ, start_response):
    """Do Fun things"""
    headers = [("Content-Type", "application/json")]
    start_response("200 OK", headers)
    form = parse_formvars(environ)
    ts = datetime.datetime.strptime(form.get("sdate"), "%Y%m%d")
    ts2 = datetime.datetime.strptime(form.get("edate"), "%Y%m%d")

    mckey = f"/json/huc12data/{ts:%Y%m%d}_{ts2:%Y%m%d}/v2"
    mc = Client("iem-memcached:11211")
    res = mc.get(mckey)
    if res is None:
        res = do(ts, ts2)
        mc.set(mckey, res, 3600)
    else:
        res = res.decode("utf-8")
    mc.close()
    return [res.encode("ascii")]


import os
import warnings
import logging

import rasterio as rio
import datetime

import admin.constants as const

from process.support import process_by_watershed_or_basin
from admin.color_ramp import color_ramp
import admin.object_store_util

# from osgeo import gdal
import multiprocessing
from glob import glob
from rasterio.warp import calculate_default_transform, reproject, Resampling
from rasterio.merge import merge
from typing import List

import admin.snow_path_lib

LOGGER = logging.getLogger(__name__)
snow_path = admin.snow_path_lib.SnowPathLib()
ostore_util = admin.object_store_util.OStore()

# Suppress warning for GCP/RPC - inquiry does not affect workflow
warnings.filterwarnings("ignore", category=rio.errors.NotGeoreferencedWarning)

def reproject_modis(date: str, name: str, pth: str, dst_crs: str):
    """Reproject modis into the target CRS

    Creates data in the
    TOP ./intermediate_tif/modis/2023.03.23/MOD10A1.A2023082.h12v02.061.2023084034450_EPSG4326.tif'

    Parameters
    ----------
    name : str
        A name string of the granule to be reprojected
        to set up intermediate files
    date : str
        The aquisition date of the granule to be reprojected
        to set up intermediate files in format YYYY.MM.DD
    pth : str
        A string path to the granule
    dst_crs : str
        The destination CRS to be reprojected to
    """
    # OMG really!!!  Could you make this any more cryptic!!! (see commented out line below)
    # pth_file_noext = ".".join(os.path.split(pth)[-1].split(".")[:-1])
    # pth example data: ./data/modis-terra/MOD10A1.061/2023.03.21/MOD10A1.A2023080.h10v02.061.2023082033825.hdf

    pth_file_noext = snow_path.file_name_no_suffix(pth)
    intermediate_tif = snow_path.get_modis_reprojected_tif(
        input_source_path=pth, date_str=date, projection=dst_crs
    )
    LOGGER.debug(f"intermediate_tif: {intermediate_tif}")
    LOGGER.debug(f"pth: {pth}")
    if not os.path.exists(intermediate_tif):
        LOGGER.debug(f"processing the modis granule: {pth_file_noext}")
        with rio.open(pth, "r") as modis_scene:
            with rio.open(modis_scene.subdatasets[0], "r") as src:
                # transform raster to dst_crs------
                transform, width, height = calculate_default_transform(
                    src.crs,
                    dst_crs,
                    src.width,
                    src.height,
                    *src.bounds,
                    resolution=const.MODIS_EPSG4326_RES,
                )
                kwargs = src.meta.copy()
                kwargs.update(
                    {
                        "driver": "GTiff",
                        "crs": dst_crs,
                        "transform": transform,
                        "width": width,
                        "height": height,
                    }
                )
                # Write reprojected granule into GTiff format
                with rio.open(intermediate_tif, "w", **kwargs) as dst:
                    reproject(
                        source=rio.band(src, 1),
                        destination=rio.band(dst, 1),
                        src_transform=src.transform,
                        src_crs=src.crs,
                        dst_transform=transform,
                        dst_crs=dst_crs,
                        resampling=Resampling.nearest,
                    )
                # -------------------------------------

def create_modis_mosaic(
    input_granule_directory: str, output_mosaic_tif: str, tifs_to_mosaic
):
    """
    Create a mosaic of all downloaded and reprojected tiffs

    Parameters
    ----------
    pth : str
        Path to directory of tiffs to be mosaic'ed

    """
    # example path: './data/intermediate_tif/modis/2023.03.21'
    LOGGER.debug(f"pth: {input_granule_directory}")
    date = os.path.basename(input_granule_directory)  # Get date var from path
    # TODO: Evalute if this is working correctly...
    #       the tifs_to_mosaic includes values like:
    #           './data/intermediate_tif/modis/2023.03.21/MOD10A1.A2023080.h12v02.061.2023082034547_EPSG4326.tif'
    #       but also includes the following tif
    #           './data/intermediate_tif/modis/2023.03.21/modis_composite_2023.03.21_2023.03.20_2023.03.19_2023.03.18_2023.03.17.tif'
    #
    # output_mosaic_tif = snow_path.get_output_modis_path(date)
    if not os.path.exists(output_mosaic_tif):
        LOGGER.debug(f"example of single file to mosaic: {tifs_to_mosaic[0]}")
        if tifs_to_mosaic:
            src_files_to_mosaic = []
            # creating a list of rasterio / rio file handles
            for f in tifs_to_mosaic:
                LOGGER.debug(f"adding to mosaic: {f}")
                src = rio.open(f, "r")
                src_files_to_mosaic.append(src)
            # Merge all granule tiffs into one
            mosaic, out_trans = merge(
                src_files_to_mosaic, bounds=[*const.BBOX], res=const.MODIS_EPSG4326_RES
            )
            out_meta = src.meta.copy()
            out_meta.update(
                {
                    "driver": "GTiff",
                    "height": mosaic.shape[1],
                    "width": mosaic.shape[2],
                    "transform": out_trans,
                }
            )
            # Write mosaic to disk
            # out_pth = os.path.join(
            #     const.OUTPUT_TIF_MODIS, date.split(".")[0], f"{date}.tif"
            # )
            output_mosaic_tif = snow_path.get_output_modis_path(date)
            LOGGER.debug(f"out_pth: {output_mosaic_tif}")
            try:
                LOGGER.debug(f"out_pth: {output_mosaic_tif}")
                # os.makedirs(os.path.split(out_pth)[0])
                dir2Create = os.path.dirname(output_mosaic_tif)
                if not os.path.exists(dir2Create):
                    os.makedirs(dir2Create)
                    LOGGER.debug(f"creating dir: {dir2Create}")
            except Exception as e:
                LOGGER.debug(e)
            LOGGER.debug(f"creating: {output_mosaic_tif}")
            with rio.open(output_mosaic_tif, "w", **out_meta) as dst:
                dst.write(mosaic)
            # Close all open tiffs that were mosaic'ed
            for f in src_files_to_mosaic:
                f.close()

def composite_mosaics(startdate: str, dates: list, out_pth: str):
    """
    Create a composite GTiff of the mosaics of a given range
    provided in the list of dates

    Parameters
    ----------
    dates : list
        Dates of mosaics to consider for compositing
    """
    # TODO: rework so inputs and outputs are fed as args
    LOGGER.debug(f"startdate: {startdate}")
    base = snow_path.get_modis_int_tif_dir(date=startdate)
    if not os.path.exists(base):
        os.makedirs(base)
        LOGGER.debug(f"created the directory {base}")
    if not os.path.exists(out_pth):
        LOGGER.debug(f"out_pth: {out_pth}")
        mosaics = []
        for date in dates:
            # tif_by_date = os.path.join(const.OUTPUT_TIF_MODIS, startdate.split(".")[0],
            #                f"{date}.tif")
            tif_by_date = snow_path.get_output_modis_path(date=date)
            mosaics.append(tif_by_date)
        with rio.open(mosaics[0], "r") as src:
            meta = src.meta.copy()
            data = src.read(1)
            for i in range(1, len(mosaics)):
                with rio.open(mosaics[i], "r") as m:
                    try:
                        data_b = m.read(1)
                        mask = (data > 100) & (data_b <= 100)
                        data[mask] = data_b[mask]
                    except Exception as e:
                        LOGGER.error(e)
                        continue
            with rio.open(out_pth, "w", **meta) as dst:
                dst.write(data, indexes=1)

def distribute(func, args):
    # Multiprocessing support to manage Pool scope
    with multiprocessing.Pool(6) as p:
        p.starmap(func, args)

def get_datespan(date: str, days: int) -> List[str]:
    """
    Build up date reference list for 5 or 8 day processing

    Parameters
    ----------
    date : str
        Starting date
    days: int
        Number of days to consider when building reference list (5 or 8)
    Returns
    ----------
    date_query: List
        Reference of all dates to consider when processing reprojetion and
        mosaic
    """
    datelist = date.split(".")
    pydate = datetime.date(int(datelist[0]), int(datelist[1]), int(datelist[2]))
    fmt_date = lambda x: x.strftime("%Y.%m.%d")
    date_query = [date]
    for d in range(1, days):
        date_query.append(fmt_date(pydate - datetime.timedelta(days=d)))
    return date_query

def clean_intermediate(date):
    residual_files = glob(os.path.join(const.INTERMEDIATE_TIF_MODIS, date, "*.tif"))
    if residual_files:
        LOGGER.info("Cleaning up residual files...")
        for f in residual_files:
            LOGGER.debug(f"delete: {f}")
            os.remove(f)

def process_modis(startdate, days):
    """
    Main trigger for processing modis from HDF4 -> GTiff and
    then clipping to watersheds/basins

    Parameters
    ----------
    startdate : str
        The startdate which modis process will base it's 5 or 8 day processing
        from
    days : int
        Number of days to process raw HDF5 granules into mosaic -> composites
        before clipping to watersheds/basins. days = 5 or days = 8 only.
    """
    LOGGER.info("MODIS Process Started")
    # bc_albers = "EPSG:3153"
    dst_crs = "EPSG:4326"

    dates = get_datespan(startdate, days)

    # pull the date composite output if it exists
    pull_date_composite(datestr_list=dates, start_date=startdate)

    # pull watershed and basin processed data if exists
    pull_watershed_basin_data(
        start_date=startdate,
        sat='modis',
        wat_basin='watersheds',
        process_async=True)
    pull_watershed_basin_data(
        start_date=startdate,
        sat='modis',
        wat_basin='basins',
        process_async=True)

    # this function recieves a start date and a days arg.
    # the days tell it how many days back to process.
    #
    # iterates over each day creating a composite tif that combines all
    # the granules for that day.

    for date in dates:
        # for each date will:
        #  - create reprojected tif to EPSG:4326 in
        #      the intermediate tif directory for each granule
        #  - then mosaics all the granules together into

        # intTif = os.path.join(const.INTERMEDIATE_TIF_MODIS, date)
        # intTif is the local path for the intermediate tif directory
        int_tif_dir = snow_path.get_modis_int_tif_dir(date)
        LOGGER.debug(f"intTif: {int_tif_dir}")
        if not os.path.exists(int_tif_dir):
            os.makedirs(int_tif_dir)
            LOGGER.debug(f"created folder: {int_tif_dir}")
        # modis_granules = glob(os.path.join(pth, date,'*.hdf'))
        # gets the granules from what exists locally after the download step
        modis_granules = snow_path.get_modis_granules(date)
        # why delete these files?  why not pick up where left off?
        # commenting out, no need to delete
        # clean_intermediate(date)

        LOGGER.info(f"REPROJ GRANULES: {date}")
        reproj_args = []
        for gran in modis_granules:
            try:
                # name = os.path.split(gran)[-1]
                name = os.path.basename(gran)
                reproj_args.append((date, name, gran, dst_crs))
            except Exception as e:
                LOGGER.error(f"Could not append {gran} : {e}")
                continue

        # if the data already exists in ostore then pull it from there
        pull_modis_data(reproj_args)

        LOGGER.debug("doing reprojections to EPSG:4326")

        # DEBUGGING... does same as distribute call but in sync.  useful for debugging
        #              Comment out for prod
        # for args in reproj_args:
        #     reproject_modis(*args)
        distribute(reproject_modis, reproj_args)

        LOGGER.info(f"CREATING MOSAICS: {date}")
        # os.path.join(const.INTERMEDIATE_TIF_MODIS,date)
        # creates the mosaic files:
        # ./data/norm/mosaics/modis/2023/<processing date>
        # example
        # ./data/norm/mosaics/modis/2023/2023.03.22.tif
        output_mosaic_tif = snow_path.get_output_modis_path(date)
        files_to_mosaic = snow_path.get_modis_intermediate_tifs(date)
        create_modis_mosaic(int_tif_dir, output_mosaic_tif, files_to_mosaic)

    LOGGER.info("COMPOSING MOSAICS INTO ONE TIF")
    # creates:
    # './data/intermediate_tif/modis/2023.03.23/modis_composite_2023.03.23_2023.03.22_2023.03.21_2023.03.20_2023.03.19.tif'
    composite_mosaic_path = snow_path.get_modis_composite_mosaic_file_name(
        start_date=startdate, date_list=dates
    )
    composite_mosaics(startdate, dates, composite_mosaic_path)
    color_ramp(composite_mosaic_path)  # just adds the color ramp to the output file

    # creates the watershed/basin clipped versions of the composite mosaic
    # in both EPSG4326 and EPSG3153
    for task in ["watersheds", "basins"]:
        LOGGER.info(f"CREATING {task.upper()}")
        # pull the 10y 20y data from object storage
        # send the dates along
        process_by_watershed_or_basin("modis", task, startdate, dates)

def pull_modis_epsg4326(local_file_list, process_async=False):
    """checks to see if the output files associated with the reproject step exist in
    object storage and if so then pulls those files from object storage vs. reprocessing
    them.

    :param local_file_list: list of the original source files that were downloaded from
        snow and ice data center.

        example path: './data/modis-terra/MOD10A1.061/2023.03.24/MOD10A1.A2023083.h09v03.061.2023085030249.hdf'
    :type local_file_list: list of files that should be created by the process/reproject
        step.
    :param process_async: indicates whether to process as syncronous or async process
    :type process_async: boolean
    """
    ostore_dir_file_cache = {}  # a dictionary to cache list queries
    arg_list = []

    for local_file in local_file_list:
        orig_dir, original_file = os.path.split(local_file)
        date_str = os.path.basename(orig_dir)
        modis_reproj_file = snow_path.get_modis_reprojected_tif(
            input_source_path=local_file, date_str=date_str
        )
        arg_list.append(modis_reproj_file)
    if process_async:
        LOGGER.debug("args")
        with multiprocessing.pool.ThreadPool(6) as p:
            p.map(ostore_util.get_file_if_exists, arg_list)

    else:
        for arg in arg_list:
            ostore_util.get_file_if_exists(arg)

        # if not os.path.exists(modis_reproj_file):
        #     if ostore_util.ostore_file_exists(local_path=modis_reproj_file):
        #         # file is in ostore so pull it
        #         LOGGER.debug(f"pulling {local_file} from ostore")
        #         ostore_util.get_file(local_file)

def pull_mosaics(datestr_list, process_async=True):
    """gets list of date strings, calculates the path that coresponsds with the mosaics
    for those dates, checks to see if they exist locally, and if they do not then looks
    to object storage bucket to get them.

    :param datestr_list: _description_
    :type datestr_list: _type_
    """
    # data/norm/mosaics/modis/2023/2023.03.22.tif
    arg_list = []

    for date in datestr_list:
        mosaic_dir = snow_path.get_mosaic_dir(sat="modis", date=date)
        mosaic_file = snow_path.get_mosaic_file(sat="modis", date=date)
        # TODO get the object store path, test to see if file is in object store, if
        # yes then pull it
        arg_list.append(mosaic_file)
    if process_async:
        with multiprocessing.pool.ThreadPool(6) as p:
            p.map(ostore_util.get_file_if_exists, arg_list)
    else:
        for arg in arg_list:
            ostore_util.get_file_if_exists(arg)

def pull_date_composite(datestr_list, start_date: str):
    """gets a list of date strings in the format YYYY.MM.DD, calculates the
    corresponding output file, tests to see if it already exists, if it does not then
    check to see if it exists in object storage.  If it does then it gets pulled down
    to the local path.

    :param datestr_list: _description_
    :type datestr_list: _type_
    """
    composite_mosaic_path = snow_path.get_modis_composite_mosaic_file_name(
        start_date=start_date, date_list=datestr_list
    )
    if not os.path.exists(composite_mosaic_path):
        if ostore_util.ostore_file_exists(composite_mosaic_path):
            ostore_util.get_file(composite_mosaic_path)

def pull_watershed_basin_data(start_date, sat, wat_basin, process_async=False):
    # data/watersheds/Stikine/modis/2023.03.23/Stikine_modis_2023.03.23_EPSG4326.tif
    # data/watersheds/Northwest/modis/2023.03.23/Northwest_modis_2023.03.23_EPSG4326.tif'
    # TODO: find references to the string EPSG:4326 and EPSG:3153 and replace with a
    #       reference to a constant from constants.
    arg_list = []
    wat_basin_names = snow_path.get_watershed_basin_list(wat_bas=wat_basin)
    for wat_basin_name in wat_basin_names:
        for projection in ['EPSG:4326', 'EPSG:3153']:
            local_path = snow_path.get_watershed_or_basin_path(
                start_date=start_date,
                sat=sat,
                watershed_basin=wat_basin,
                watershed_name=wat_basin_name,
                projection=projection
            )
            arg_list.append(local_path)
    if process_async:
        with multiprocessing.pool.ThreadPool(6) as p:
            p.map(ostore_util.get_file_if_exists, arg_list)
    else:
        for arg in arg_list:
            ostore_util.get_file_if_exists(arg)

def pull_modis_data(reproj_args):
    # pulling the files that generated by the reprojection step
    local_files = [arg[2] for arg in reproj_args]
    pull_modis_epsg4326(local_files, process_async=True)

    # pulling the mosaics of all the granules
    dates = list(set([arg[0] for arg in reproj_args]))
    pull_mosaics(dates, process_async=False)


from scipy.misc import imread, imresize
import numpy as np

from scipy.misc import imread
import matplotlib.pyplot as plt

# Helper functions to deal with image preprocessing
from lib.utils.image_utils import load_image, preprocess_image, deprocess_image

def get_session():
    """Create a session that dynamically allocates memory."""
    # See: https://www.tensorflow.org/tutorials/using_gpu#allowing_gpu_memory_growth
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    session = tf.Session(config=config)
    return session

def rel_error(x,y):
    return np.max(np.abs(x - y) / (np.maximum(1e-8, np.abs(x) + np.abs(y))))

#-------------------------------------Finish------------------------------------

from lib.classifiers.squeezenet import SqueezeNet
import tensorflow as tf

tf.reset_default_graph() # remove all existing variables in the graph
sess = get_session() # start a new Session

# Load pretrained SqueezeNet model
SAVE_PATH = 'lib/datasets/squeezenet.ckpt'
model = SqueezeNet(save_path=SAVE_PATH, sess=sess)

# Load data for testing
content_img_test = preprocess_image(load_image('styles/tubingen.jpg', size=192))[None]
style_img_test = preprocess_image(load_image('styles/starry_night.jpg', size=192))[None]

def content_loss(content_weight, content_current, content_original):
    """
    Compute the content loss for style transfer.

    Inputs:
    - content_weight: scalar constant we multiply the content_loss by.
    - content_current: features of the current image, Tensor with shape [1, height, width, channels]
    - content_target: features of the content image, Tensor with shape [1, height, width, channels]

    Returns:
    - scalar content loss
    """
    loss = content_weight * tf.reduce_sum((content_current - content_original)**2)
    return loss

def gram_matrix(features, normalize=True):
    """
    Compute the Gram matrix from features.

    Inputs:
    - features: Tensor of shape (1, H, W, C) giving features for
      a single image.
    - normalize: optional, whether to normalize the Gram matrix
        If True, divide the Gram matrix by the number of neurons (H * W * C)

    Returns:
    - gram: Tensor of shape (C, C) giving the (optionally normalized)
      Gram matrices for the input image.
    """
    shape = tf.shape(features)
    features_reshaped = tf.reshape(features, (shape[1]*shape[2], shape[3]))
    gram = tf.matmul(tf.transpose(features_reshaped), features_reshaped)
    if normalize:
        gram /= tf.cast((shape[3] * shape[1] * shape[2]), tf.float32)
    return gram

def style_loss(feats, style_layers, style_targets, style_weights):
    """
    Computes the style loss at a set of layers.

    Inputs:
    - feats: list of the features at every layer of the current image, as produced by
      the extract_features function.
    - style_layers: List of layer indices into feats giving the layers to include in the
      style loss.
    - style_targets: List of the same length as style_layers, where style_targets[i] is
      a Tensor giving the Gram matrix the source style image computed at
      layer style_layers[i].
    - style_weights: List of the same length as style_layers, where style_weights[i]
      is a scalar giving the weight for the style loss at layer style_layers[i].

    Returns:
    - style_loss: A Tensor contataining the scalar style loss.
    """
    # Hint: you can do this with one for loop over the style layers, and should
    # not be very much code (~5 lines). You will need to use your gram_matrix function.
    total_loss = 0.0
    for i in range(len(style_layers)):
        G = style_targets[i]
        A = gram_matrix(feats[style_layers[i]])
        total_loss += style_weights[i]*tf.reduce_sum((G - A)**2)

    return total_loss

def tv_loss(img, tv_weight):
    """
    Compute total variation loss.

    Inputs:
    - img: Tensor of shape (1, H, W, 3) holding an input image.
    - tv_weight: Scalar giving the weight w_t to use for the TV loss.

    Returns:
    - loss: Tensor holding a scalar giving the total variation loss
      for img weighted by tv_weight.
    """
    # Your implementation should be vectorized and not require any loops!
    left_loss = tf.reduce_sum((img[:, 1:, :, :] - img[:, :-1, :, :])**2)
    down_loss = tf.reduce_sum((img[:, :, 1:, :] - img[:, :, :-1, :])**2)
    loss = tv_weight*(left_loss + down_loss)
    return loss

def style_transfer_loss(content_image, style_image, image_size, style_size, content_layer, content_weight,
                   style_layers, style_weights, tv_weight, init_random):
    """

    Inputs:
    - content_image: filename of content image
    - style_image: filename of style image
    - image_size: size of smallest image dimension (used for content loss and generated image)
    - style_size: size of smallest style image dimension
    - content_layer: layer to use for content loss
    - content_weight: weighting on content loss
    - style_layers: list of layers to use for style loss
    - style_weights: list of weights to use for each layer in style_layers
    - tv_weight: weight of total variation regularization term
    - init_random: initialize the starting image to uniform random noise
    """
    # Extract features from the content image
    content_img = preprocess_image(load_image(content_image, size=image_size))
    feats = model.extract_features(model.image)
    content_target = sess.run(feats[content_layer],
                              {model.image: content_img[None]})

    # Initialize generated image to content image

    if init_random:
        img_var = tf.Variable(tf.random_uniform(content_img[None].shape, 0, 1), name="image")
    else:
        img_var = tf.Variable(content_img[None], name="image")

    # Extract features from the style image
    style_img = preprocess_image(load_image(style_image, size=style_size))
    style_feat_vars = [feats[idx] for idx in style_layers]
    style_target_vars = []
    # Compute list of TensorFlow Gram matrices
    for style_feat_var in style_feat_vars:
        style_target_vars.append(gram_matrix(style_feat_var))
    # Compute list of NumPy Gram matrices by evaluating the TensorFlow graph on the style image
    style_targets = sess.run(style_target_vars, {model.image: style_img[None]})

    # Extract features on generated image
    feats = model.extract_features(img_var)
    # Compute loss
    c_loss = content_loss(content_weight, feats[content_layer], content_target)
    s_loss = style_loss(feats, style_layers, style_targets, style_weights)
    t_loss = tv_loss(img_var, tv_weight)
    loss = c_loss + s_loss + t_loss

    return loss, img_var, content_img, style_img

def style_transfer_train(loss, img_var, initial_lr=3.0, decayed_lr=0.1, decay_lr_at=180, max_iter=200, print_every=50):
    # Create and initialize the Adam optimizer
    lr_var = tf.Variable(initial_lr, name="lr")
    # Create train_op that updates the generated image when run
    with tf.variable_scope("optimizer") as opt_scope:
        train_op = tf.train.AdamOptimizer(lr_var).minimize(loss, var_list=[img_var])
    # Initialize the generated image and optimization variables
    opt_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=opt_scope.name)
    sess.run(tf.variables_initializer([lr_var, img_var] + opt_vars))
    # Create an op that will clamp the image values when run
    clamp_image_op = tf.assign(img_var, tf.clip_by_value(img_var, -1.5, 1.5))

    imgs_in_process = []

    # Hardcoded handcrafted
    for t in range(max_iter):
        # Take an optimization step to update img_var
        sess.run(train_op)
        if t < decay_lr_at:
            sess.run(clamp_image_op)
        if t == decay_lr_at:
            sess.run(tf.assign(lr_var, decayed_lr))
        if t % print_every == 0:
            print("train step: %d" % t)
            img = sess.run(img_var)
            imgs_in_process.append(img[0])
    print("train step: %d" % t)
    final_img = sess.run(img_var)[0]
    return imgs_in_process, final_img

# Set up optimization hyperparameters
initial_lr = 3.0
decayed_lr = 0.1
decay_lr_at = 450
max_iter = 500
print_every = 50
# Composition VII + Tubingen
params1 = {
    'content_image' : 'styles/goldengate.jpeg',
    'style_image' : 'styles/starry_night.jpg',
    'image_size' : 192,
    'style_size' : 512,
    'content_layer' : 3,
    'content_weight' : 5e-2,
    'style_layers' : (1, 4, 6, 7),
    'style_weights' : (50000, 800, 12, 1),
    'tv_weight' : 5e-2,
    'init_random' : False
}

loss, img_var, content_img, style_img = style_transfer_loss(**params1)
imgs_in_process, final_img = style_transfer_train(loss, img_var, initial_lr, decayed_lr, decay_lr_at, max_iter, print_every)

#show original image
f, axarr = plt.subplots(1,2)
axarr[0].axis('off')
axarr[1].axis('off')
axarr[0].set_title('Content Source Img.')
axarr[1].set_title('Style Source Img.')
axarr[0].imshow(deprocess_image(content_img))
axarr[1].imshow(deprocess_image(style_img))
plt.show()

#show image in process
f, axarr = plt.subplots(1,len(imgs_in_process))
for i in range(len(imgs_in_process)):
    current_step = i * print_every + 1
    current_img = imgs_in_process[i]
    axarr[i].axis('off')
    axarr[i].set_title("Iteration %d" % current_step)
    axarr[i].imshow(deprocess_image(current_img, rescale=True))
plt.show()

#show final result
current_img = final_img
plt.imshow(deprocess_image(final_img, rescale=True))
plt.axis('off')
plt.show()