import logging

from edd import receiver
from main import signals

from . import models

logger = logging.getLogger(__name__)

@receiver(signals.study_created)
def log_study_created(sender, study, user, **kwargs):
    logger.debug(f"Got signal study_created from {sender}")
    models.StudyLog.objects.create(
        event=models.StudyLog.Event.CREATED, study=study, user=user,
    )

@receiver(signals.study_described)
def log_study_described(sender, study, user, count, **kwargs):
    logger.debug(f"Got signal study_described from {sender}")
    models.StudyLog.objects.create(
        detail={"count": count},
        event=models.StudyLog.Event.DESCRIBED,
        study=study,
        user=user,
    )

@receiver(signals.study_exported)
def log_study_exported(sender, study, user, count, cross, **kwargs):
    logger.debug(f"Got signal study_exported from {sender}")
    models.StudyLog.objects.create(
        detail={"count": count},
        event=models.StudyLog.Event.EXPORTED,
        study=None if cross else study,
        user=user,
    )

@receiver(signals.study_imported)
def log_study_imported(sender, study, user, protocol, count, **kwargs):
    logger.debug(f"Got signal study_imported from {sender}")
    models.StudyLog.objects.create(
        detail={"count": count, "protocol": protocol.uuid},
        event=models.StudyLog.Event.IMPORTED,
        study=study,
        user=user,
    )

@receiver(signals.study_permission_change)
def log_study_permission_change(
    sender, study_id, user, permission, selector, applied, **kwargs
):
    logger.debug(f"Got signal study_permission_change from {sender}")
    # even if study is getting deleted, this will run *before* deletion
    study = models.StudyLog.lookup_study(study_id)
    detail = {"permission": permission, "slug": study.slug}
    if applied:
        detail.update(added=selector)
    else:
        detail.update(removed=selector)
        # don't allow setting foreign key ref
        # removal could be from deleting the study + cascade to permission
        study = None
    models.StudyLog.objects.create(
        detail=detail, event=models.StudyLog.Event.PERMISSION, study=study, user=user,
    )

@receiver(signals.study_viewed)
def log_study_viewed(sender, study, user, **kwargs):
    logger.debug(f"Got signal study_viewed from {sender}")
    models.StudyLog.objects.create(
        event=models.StudyLog.Event.VIEWED, study=study, user=user,
    )

@receiver(signals.study_worklist)
def log_study_worklist(sender, study, user, count, cross, **kwargs):
    logger.debug(f"Got signal study_worklist from {sender}")
    models.StudyLog.objects.create(
        detail={"count": count},
        event=models.StudyLog.Event.WORKLIST,
        study=None if cross else study,
        user=user,
    )


########################################################
########################################################

##INSERTS INTO index.py's dedicated space

import random

greetings = ["hey",  "hello", "hi", "hello there", "hi there", "hey there", "hi", "greetings"]

@client.event
async def on_message(message):

    if message.author == client.user:
        return

    message.content = message.content.lower()

    def client_replies(content):
        content = random.choice(content)
        #If you want to log messages from its DMs
        if config.getboolean('SETTINGS', 'debug'):
            if message.guild is None:
                print(current_time() + "-------DM------- \n" + message.author.name + ": " + message.content)
            else:
                print(f"{client_name}: {content}")
                print("----------------")
                print(f'To: {message.author.name} \n{current_time()}{client_name}: {content}\n ----------------')

        randomNum = random.randint(0, 2)
        if randomNum == 0:
            return message.channel.send(content)
        elif randomNum == 1:
            return message.channel.send(f"{message.author.mention} {content}")
        elif randomNum == 2:
            return message.reply(content)

    if client.user.mentioned_in(message) or message.guild is None:
        print(message.content)
        if any(word in message.content for word in greetings):
            await client_replies(greetings)


import findspark
import pyspark
from pyspark.sql import SparkSession
from pyspark.sql.functions import *
from pyspark.sql.types import *
from pyspark import SparkConf, SparkContext
import os
import sys
import py4j
import json

findspark.init()
os.environ['JAVA_HOME'] = '/usr/local/java/jdk-11.0.18+10'
os.environ['SPARK_HOME'] = '/usr/local/spark'
sys.path.append('/usr/local/spark/python')
sys.path.append('/usr/local/spark/python/lib/pyspark.zip')
sys.path.append('/usr/local/spark/python/lib/py4j-0.10.9.5-src.zip')
sys.path.append('/usr/local/spark/bin')
sys.path.append('/usr/local/spark/yarn/spark-3.3.2-yarn-shuffle.jar')
sys.path.append('/usr/local/spark/jars')

# Create a Spark session
spark = SparkSession.builder \
    .config("spark.driver.host", "localhost") \
    .config("spark.driver.logLevel", "DEBUG") \
    .appName('Realtime_Streamprocessing_app') \
    .getOrCreate()

# Define the schema for the data
schema = StructType([
    StructField("id", IntegerType(), True),
    StructField("timestamp", LongType(), True),
    StructField("distance_covered", DoubleType(), True),
    StructField("engine_speed", DoubleType(), True),
    StructField("fuel_consumed", DoubleType(), True),
])

# Consume data from the Kafka topic
df = spark \
    .readStream \
    .format('kafka') \
    .option('kafka.bootstrap.servers', 'localhost:9092') \
    .option('subscribe', 'Truck-Data') \
    .option('startingOffsets', 'earliest') \
    .option('failOnDataLoss', 'false') \
    .option('includeTimestamp', 'true') \
    .option('schema', schema) \
    .load()

# Define the schema for the data
schema = StructType([
    StructField('id', IntegerType(), True),
    StructField('timestamp', StringType(), True),
    StructField('distance_covered', DoubleType(), True),
    StructField('engine_speed', DoubleType(), True),
    StructField('fuel_consumed', DoubleType(), True),
])

# Convert the value column from binary to string, and parse the fields
parsed_df = df.selectExpr('CAST(value AS STRING)').select(from_json('value', schema).alias('data'), 'timestamp')

# Flatten the struct column and rename the fields
transformed_df = parsed_df.selectExpr(
    'data.id AS id',
    'timestamp',
    'data.distance_covered AS distance_covered',
    'data.engine_speed AS engine_speed',
    'data.fuel_consumed AS fuel_consumed'
)

# Now, we write our transformed data to a MySQL table
transformed_df.writeStream \
    .format("jdbc") \
    .option("url", "jdbc:mysql://MySQL_Container:3306/KAFKA_DB") \
    .option("dbtable", "SPARK_TABLE_TRANSFORMED") \
    .option("user", "root") \
    .option("password", "root") \
    .option("checkpointLocation", "/tmp/checkpoints") \
    .option("driver", "com.mysql.cj.jdbc.Driver") \
    .start()

# Submit my file to spark-submit

# spark.sparkContext.addPyFile('./spark_streamer.py')
# spark.sparkContext.submit(MainClass='spark_streamer.py')


import torch
from torch import nn

class Simple(nn.Module):

    '''
    A simple image2image CNN.
    '''
    def __init__(self):
        # This line is very important!
        super().__init__()

        self.conv1 = nn.Conv2d(3, 64, 3)
        self.relu1 = nn.ReLU(inplace=True)
        self.conv2 = nn.Conv2d(64, 64, 3)
        self.relu2 = nn.ReLU(inplace=True)
        self.conv3 = nn.Conv2d(64, 3, 3)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        '''
        SimpleNet forward function.

        Args:
            x (B x 3 x H x W Tensor): An input image

        Return:
            (B x 3 x H x W Tensor): An output image
        '''
        # Autograd will keep the history.
        x = self.conv1(x)
        x = self.relu1(x)
        x = self.conv2(x)
        x = self.relu2(x)
        x = self.conv3(x)
        return x


x_ = {'user1': [213, 213, 213, 15, 213],
           'user2': [54, 54, 119, 119, 119],
           'user3': [213, 98, 98, 35]}
def set_func(x):
    ids = x

    set_list = []

    for value in ids.values():
      for num in value:
        set_list.append(num)
    return set(set_list)

y_ = {'facebook': 55, 'yandex': 120, 'vk': 115, 'google': 99, 'email': 42, 'ok': 98}
def max_key(y):
    stats = y

    max = 0
    key_max = ''

    for key, value in stats.items():
      if value > max:
        max = value
        key_max = key

    return key_max

z_ = ['2018-01-01', 'yandex', 'cpc', 100]
def foo(z):
    test_list = z
    test_dict = test_list[-1]

    for elem in test_list[-2::-1]:
        test_dict = {elem: test_dict}

    return test_dict


"""Test module for the :mod:`gwgen.main` module"""
import os.path as osp
import unittest
from gwgen.utils import file_len
import _base_testing as bt

class OrganizerTest(bt.BaseTest):
    """Test the :class:`gwgen.main.ModuleOrganizer` class"""

    def test_compile_model(self):
        """Test the compilation of a model"""
        self.organizer.setup(self.test_dir)
        projectname = self.organizer.projectname
        self.organizer.compile_model()
        binpath = osp.join(self.test_dir, projectname, 'bin', 'weathergen')
        self.assertTrue(osp.exists(binpath),
                        msg='binary %s does not exist!' % binpath)

    def test_run(self):
        self._test_init()
        organizer = self.organizer
        exp = organizer.experiment
        self.organizer.run(ifile=osp.join(
            bt.test_root, 'test_data', 'input.csv'))
        fname = osp.join(self.test_dir, organizer.projectname, 'experiments',
                         exp, 'outdata', exp + '.csv')
        self.assertTrue(osp.exists(fname), msg='Output file %s not found!' % (
            fname))
        nlines = file_len(fname)
        self.assertGreater(nlines, 2, msg='No output generated!')

    def test_wind_bias_correction(self):
        """Test gwgen bias wind"""
        self._test_init()
        self.organizer.exp_config['reference'] = osp.join(
            bt.test_root, 'test_data', 'long_reference.csv')
        fname, ext = osp.splitext(self.stations_file)
        self.stations_file = osp.join(bt.test_root, 'test_stations_long.dat')
        self.organizer.exp_config['eval_stations'] = self.stations_file
        ifile = osp.join(bt.test_root, 'test_data', 'long_input.csv')
        self.organizer.exp_config['namelist'] = {
            'weathergen_ctl': {'wind_bias_coeffs': [1.0] + [0.0] * 5,
                               'wind_intercept_bias_a': -9999.,
                               'wind_intercept_bias_b': -9999.},
            'main_ctl': {}}
        self.organizer.parse_args(['run',  '-i', ifile])
        self.organizer.parse_args('bias -q 1-100-5,99 wind'.split())
        self.organizer.fix_paths(self.organizer.exp_config)
        ofile = self.organizer.exp_config['postproc']['bias']['wind'][
            'plot_file']
        self.assertTrue(osp.exists(ofile), msg=ofile + ' is missing')

if __name__ == '__main__':
    unittest.main()


"""
This file shows how to use the layers defined in layers.py file, read dataset, and use
my implementation of gradient descent to perform training on a dataset.
"""

import tensorflow as tf
import layers as l
import loss as lo
from extras import *
from train import *

NUMBER_EPOCHS_TO_TRAIN = 1000
BATCH_SIZE = 120

def iris_process_input_output_fn(features, labels, feature_columns):
	"""
	features: output of iterator
	labels: output of iterator
	feature_columns: list of feature columns to get features from dataset

	Returns: processed feaures and labels
	"""
	feature_batch = tf.feature_column.input_layer(features, feature_columns)
	output_batch = tf.one_hot(labels, depth=3, dtype=tf.int32)

	return feature_batch, output_batch

def make_layers(n_features):
	"""
	n_features: number of features in the dataset

	Returns: the list of layers
	"""

	hidden_layer_1_nodes = [10]
	hidden_layer_2_nodes = [3]
	n_classes = [3]

	with tf.variable_scope("input_layer_scope"):
		input_layer = l.InputLayer([n_features])

	with tf.variable_scope("hidden_layer_1_scope"):
		hidden_layer_1 = l.FullyConnectedLayer(hidden_layer_1_nodes, input_layer,
											activation=None, name="hidden layer 1")

	with tf.variable_scope("hidden_layer_2_scope"):
		hidden_layer_2 = l.FullyConnectedLayer(hidden_layer_2_nodes, hidden_layer_1,
											activation=None, name="hidden layer 2")

	with tf.variable_scope("output_layer_scope"):
		hidden_layer_3 = l.FullyConnectedLayer(n_classes, hidden_layer_2,
											activation=None, name="output layer")

	layers = [input_layer, hidden_layer_1, hidden_layer_2, hidden_layer_3]

	return layers

if __name__ == '__main__':

	batch_size = BATCH_SIZE

	train, test = load_data()
	dataset = train_input_fn(train[0], train[1], batch_size)

	test_size = len(test[0])
	test_dataset = eval_input_fn(test[0], test[1], test_size)

	keys = train[0].keys()
	feature_columns = []
	for feature_name in keys:
		feature_columns.append(tf.feature_column.numeric_column(key=feature_name))

	train_size = len(train[0])
	n_features = train[0].shape[1]

	print("size of the dataset is:", train_size)
	print("number of features are:", n_features)

	alpha = 0.01

	layers = make_layers(n_features)
	loss = lo.SoftmaxCrossEntropyLoss(layers[-1])

	kwargs = {}
	kwargs['feature_columns'] = feature_columns

	fit(iris_process_input_output_fn, kwargs, layers, loss, dataset, train_size, alpha)

	print()
	predict(iris_process_input_output_fn, kwargs, layers, test_dataset, test_size, test[1])


#!/usr/bin/env python3
# Julian Ogata
# CPSC 323-04
# 2022-5-01
# jogata@csu.fullerton.edu
#
# Calculates a user given postfix expression
#

"""Project 3: Post-Fix Program"""

import operator

class PostFix:
    '''Holds methods to calculate a given expression in postfix notation'''
    def __init__(self):
        self._operations = {'+' : operator.add,
                            '-' : operator.sub,
                            '*' : operator.mul,
                            '/' : operator.truediv,
                            '%' : operator.mod,
                            '^' : operator.pow}
        self._user_str = []
        self._stack = ['$']
        self._assigned_vals = {}

    @property
    def operations(self):
        '''Gets operation dict'''
        return self._operations

    @property
    def user_str(self):
        '''Gets user string'''
        return self._user_str

    @user_str.setter
    def user_str(self, new_str):
        '''Sets user string'''
        self._user_str = new_str

    @property
    def stack(self):
        '''Gets stack'''
        return self._stack

    @stack.setter
    def stack(self, new_stack):
        '''Sets stack'''
        self._stack = new_stack

    @property
    def assigned_vals(self):
        '''Gets the assigned values of variables dict'''
        return self._assigned_vals

    @assigned_vals.setter
    def assigned_vals(self, new_val):
        '''Sets the dict of assigned values'''
        self._assigned_vals = new_val

    def user_input(self):
        '''Asks user for an expression and variable values'''
        in_input = True
        counter = 0
        while in_input:
            expr_str = input('Enter a postfix expression with a $ at the end: ')
            if expr_str[-1] != '$':
                print('Please add a "$" at the end of the expression')
            else:
                in_input = False
        # Loops through given expression until it encounters "$"
        while expr_str[counter] != '$':
            # Check if the current variable is not an operator or has already been assigned a value
            if expr_str[counter] not in self.operations\
               and expr_str[counter] not in self.assigned_vals.keys():
                # Start assigning integer value
                while True:
                    try:
                        value = int(input('Enter the value of {}: '.format(expr_str[counter])))
                        self.user_str.append(value)
                        self.assigned_vals.update({expr_str[counter]: value})
                        break
                    except ValueError:
                        print('Please input a integer')
            # If variable already assigned, push value onto user_str
            elif expr_str[counter] in self.assigned_vals.keys():
                self.user_str.append(self.assigned_vals.get(expr_str[counter]))
            # If variable is an operator, push onto user_str
            elif expr_str[counter] in self.operations:
                self.user_str.append(expr_str[counter])
            counter += 1
        self.user_str.append('$')
        print("user str: {}".format(self.user_str))

    def translate(self):
        '''Translates the user's string, calculates values, and adds to stack'''
        counter = 0
        # Loop through user_str until "$" is encountered
        while self.user_str[0] != '$':
            # If the value is an operator, calculate integers in stack and pop
            if self.user_str[0] in self.operations:
                counter = self.calculate(self.user_str[0], counter)
                self.user_str.pop(0)
            # Append value onto stack
            else:
                self.stack.append(self.user_str[0])
                self.user_str.pop(0)
            print('Stack: {}'.format(self.stack))
            counter += 1

    def calculate(self, operation, counter):
        """Pops and calculates values in stack with given operator,
           returns the new position of the stack"""
        oper = self.operations.get(operation)
        x_var = self.stack.pop()
        y_var = self.stack.pop()
        result = oper(y_var, x_var)
        print('{} {} {} = {}'.format(y_var, operation, x_var, result))
        self.stack.append(result)
        return counter - 1

    def reset(self):
        '''Resets class variables'''
        self.user_str = []
        self.assigned_vals = {}
        self.stack = ['$']

    def run(self):
        '''Runs Program'''
        running = True
        prompt = True
        while running:
            self.user_input()
            self.translate()
            if len(self.stack) == 2:
                print('Final value = {}'.format(self.stack[-1]))
            else:
                print('Expression did not result in a final value\n\tStack: {}'
                      .format(self.stack))
            while prompt:
                conintue = input('Continue(y/n)?')
                if conintue == 'y':
                    self.reset()
                    prompt = False
                elif conintue == 'n':
                    running = False
                    prompt = False
                else:
                    print('Please input either "y" or "n"')
            prompt = True

if __name__ == '__main__':
    POST = PostFix()
    POST.run()


from __future__ import print_function
import datetime
import os
import pymongo
import pandas as pd
import math
import shapely
import geopandas as gpd
import rasterio
import rasterio.features
import rasterio.enums
import PIL.Image
import numpy as np
from geopy.distance import great_circle
import matplotlib
import rasterstats

matplotlib.use('PS')

def save_image(raster, name):
    img = PIL.Image.fromarray(
        np.array(255 * raster / raster.max(),
        dtype=np.uint8))
    img.save(name + ".png", "PNG")

world_df = gpd.read_file(os.path.join(os.path.dirname(os.path.realpath(__file__)), "world.geo.json"))
# Use cylindrical equal-area projection
# https://gis.stackexchange.com/questions/218450/getting-polygon-areas-using-geopandas
world_df = world_df.to_crs({'proj': 'cea'})

def get_airport_to_country_code(db):
    nameToISOs = {
      'North Korea': 'KP',
      'South Korea': 'KR',
      'United States Minor Outlying Islands': 'US',
      'Macau': 'MO',
      'Reunion': 'RE',
      'Christmas Island': 'CX',
      'Guadeloupe': 'GP',
      'Ivory Coast (Cote d\'Ivoire)': 'CI',
      'French Guiana': 'GF',
      'Western Samoa': 'WS',
      'Saint Vincent and Grenadines': 'VC',
      'Guinea Bissau': 'GW',
      'Cocos (Keeling) Islands': 'CC',
      'Grenada and South Grenadines': 'GD',
      'Mayotte': 'YT',
      'Martinique': 'MQ',
      'Tuvalu': 'TV',
      'Gibraltar': 'GI',
      'Bonaire, Saint Eustatius & Saba': None,
      'Palestinian Territory': None,
      'Curacao': None,
      'Unknown Country': None
    }
    name_props = [
      'name',
      'name_long',
      'formal_en',
      'name_alt',
      'name_sort',
      'formal_en',
      'brk_name'
    ]
    for idx, row in world_df.iterrows():
        for prop in name_props:
            value = row[prop]
            if value:
                nameToISOs[value] = row.iso_a2
    airport_to_country_code = {
        airport['_id']: nameToISOs[airport['countryName']]
        for airport in db.airports.find({})
    }
    return airport_to_country_code

def plot_airport(long_lat, rasterio_handle, out_raster, magnitude):
    """
    Plot the airport catchment weight function at the given
    longitude and latitude on the out_raster array
    using the transform specified by the rasterio_handle.
    The weight function has a zero to one value which is multiplied by
    the magnitude parameter.
    """
    # Pixel distance around airport where catchment portions will be computed.
    WINDOW_SIZE = (20, 20)
    # Distance decay functions and parameters from:
    # http://www.tsi.lv/sites/default/files/editor/science/Publikacii/RelStat_09/sess_8_pavlyuk.pdf
    a = -11.13
    b = 2.72
    row, col = np.array(rasterio_handle.index(*long_lat)).astype(int)
    for i in range(row - int(WINDOW_SIZE[0] / 2), row + int(WINDOW_SIZE[0] / 2)):
        for j in range(col - int(WINDOW_SIZE[1] / 2), col + int(WINDOW_SIZE[1] / 2)):
            distance_km = great_circle(
                long_lat[::-1],
                rasterio_handle.xy(i, j)[::-1]).kilometers
            value = magnitude / (1 + math.exp(a + b * math.log(distance_km, math.e)))
            out_raster[i % out_raster.shape[0], j % out_raster.shape[1]] += value

def create_projected_point(lon_lat):
    """
    Create a shaplely geometric point at the given longitude and latitude that
    is projected to cylindrical equal area coordinates.
    """
    return gpd.GeoDataFrame(
        crs={'init':'epsg:4326'},
        geometry=[shapely.geometry.Point(*lon_lat)]
    ).to_crs({'proj':'cea'}).geometry.iloc[0]

def create_location_shapes(resolved_location_tree, parent_shape=None):
    """
    Tranform a resolved value location tree into shapely geometric shapes
    that have values corresponding to their overall resolved value minus
    that of their contained shapes.
    """
    shapes = []
    for child in resolved_location_tree.get('children', []):
        new_shape = None
        shape_value = None
        country_shape = None
        location = child['location']
        matching_countries = world_df[world_df.iso_a2 == location.get('countryCode', 'NoCountry')]
        excess_value = child['value'] - sum(child2['value'] for child2 in child.get('children', []))
        # Excess value could be slightly negative due to floating point error.
        if excess_value < -0.01:
            print(child)
            print([child2['value'] for child2 in child['children']])
            raise Exception('Bad tree')
        excess_value = max(0, excess_value)
        if len(matching_countries):
            country_shape = matching_countries.geometry.iloc[0]
        if location.get('featureCode', '').startswith('PCL') and country_shape:
            new_shape = country_shape
        else:
            # Plot the location as a circle clipped by its containing location.
            lon_lat = location.get('longitude'), location.get('latitude')
            if lon_lat[0] and lon_lat[1]:
                # default area 10 square km
                shape_area = location.get('area', 10e3)
                if len(matching_countries) and 'population' in location:
                    # Set the shape area as a proportion of the country area
                    # equal to the proportion of the country's population it contains.
                    country_portion = float(location['population']) / matching_countries.pop_est.iloc[0]
                    shape_area = max(country_shape.area * country_portion, shape_area)
                new_shape = create_projected_point(lon_lat).buffer(math.sqrt(shape_area / math.pi))
                if parent_shape:
                    new_shape = new_shape.intersection(parent_shape)
                if new_shape.area == 0:
                    new_shape = None
        child_shapes = create_location_shapes(child, new_shape)
        child_shapes_out = []
        for child_shape_val in child_shapes:
            if child_shape_val[0] is None:
                # If the child location has no shape absorb it into the
                # parent location.
                excess_value += child_shape_val[1]
            else:
                child_shapes_out.append(child_shape_val)
        # # Remove overlapping child shape regions from parent shape.
        # if len(child_shapes_out) > 0:
        #     all_child_shape = child_shapes_out[0][0]
        #     for child_shape, value in child_shapes_out[1:]:
        #         all_child_shape = all_child_shape.union(child_shape)
        #     new_shape = new_shape.difference(all_child_shape)
        shapes.append((new_shape, excess_value,))
        shapes += child_shapes_out
    return shapes

def compute_case_raster(resolved_location_tree, population_raster, population_raster_data):
    """
    Convert a resolved value location tree into a raster map where the
    value of each pixel corresponds to the number of cases that occurred
    within it.
    """
    shape_values = create_location_shapes(resolved_location_tree)
    print("Removed cases:", sum([value
        for shape, value in shape_values
        if not(shape and shape.area > 0)]))
    shape_values = [shape_value
        for shape_value in shape_values
        if shape_value[0] and shape_value[0].area > 0]
    projected_shapes = list(gpd.GeoDataFrame(
        crs={'proj':'cea'},
        geometry=[shape for shape, _ in shape_values]).to_crs({
            'init':'epsg:4326'
        }).geometry)
    case_raster = np.zeros(population_raster.shape)
    if len(shape_values) > 0:
        for projected_shape, (_, value) in zip(projected_shapes, shape_values):
            # There a couple options for combining sub-locations with their
            # parents. One is to limit sub-location desitys to the cases within
            # them, replacing the parent location pixels even when
            # they have a greater density.
            # The other is to add the sub-location densitys to the parent location
            # densitys. I think adding makes more sense because data for
            # sub-locations is likely to be more incomplete.
            mask = rasterio.features.rasterize(
                [(projected_shape, 1.0,)],
                out_shape=population_raster.shape,
                transform=population_raster.transform,
                all_touched=True)
            masked_pop = population_raster_data * mask
            case_raster += masked_pop * (value / (0.1 + masked_pop.sum()))
    return case_raster

def compute_outflows(db, match_query):
    """
    Compute the total outbound seats of each airports for flights matching the
    given query.
    """
    result = {}
    for pair in db.flights.aggregate([
        {
            "$match": match_query
        }, {
            '$group': {
                '_id': "$departureAirport",
                'totalSeats': {
                    '$sum': '$totalSeats'
                }
            }
        }
    ]):
        if pair['totalSeats'] > 0:
            result[pair['_id']] = pair['totalSeats']
    return result

if __name__ == "__main__":
    import os
    import requests
    import yaml

    population_raster = rasterio.open('gpw/gpw_v4_population_count_rev10_2015_15_min.tif')
    population_raster_data = population_raster.read(1)
    population_raster_data[population_raster_data < 0] = 0

    end_date = datetime.datetime(2018, 4, 17)
    start_date = datetime.datetime(2018, 3, 29)

    results = requests.get('https://eidr-connect.eha.io/api/events-with-resolved-data', params={
        'ids': ['RJ8oAew6rntCQMuZY'],
        'startDate': start_date.isoformat(),
        'endDate': end_date.isoformat(),
        'eventType': 'auto',
        'fullLocations': True,
        'activeCases': True
    }).json()['events']

    import yaml
    print(yaml.safe_dump(results))
    db = pymongo.MongoClient(os.environ['MONGO_HOST'])['flirt']

    resolved_event_data = results[0]
    case_raster = compute_case_raster(resolved_event_data['fullLocations'], population_raster, population_raster_data)
    print('total cases:', case_raster.sum())
    actual_case_total = sum(child2['value'] for child2 in resolved_event_data['fullLocations']['children'])
    print('error:', case_raster.sum() / actual_case_total)
    save_image(case_raster ** 0.2, "case_raster")

    outflows = compute_outflows(db, {
        "departureDateTime": {
            "$lte": end_date,
            "$gte": start_date
        }
    })
    max_outflow = max(outflows.values())

    all_airport_raster_data = np.zeros(population_raster.shape)
    for airport in list(db.airports.find()):
        magnitude = float(outflows.get(airport['_id'], 0)) / max_outflow
        if magnitude > 0:
            plot_airport(
                airport['loc']['coordinates'],
                population_raster, all_airport_raster_data,
                magnitude)
    save_image(all_airport_raster_data, "all_airport_raster")

    rows = []
    for airport in db.airports.find():
        result = np.zeros(population_raster_data.shape)
        magnitude = float(outflows.get(airport['_id'], 0)) / max_outflow
        if magnitude > 0:
            plot_airport(
                airport['loc']['coordinates'],
                population_raster, result,
                magnitude)
            result[all_airport_raster_data > 0] = result[all_airport_raster_data > 0] / all_airport_raster_data[all_airport_raster_data > 0]
            if airport['_id'] == 'SEA':
                save_image(result, 'SEA_airport_raster')
            airport['catchment_cases'] = (result * case_raster).sum()
            airport['catchment_pop'] = (result * population_raster_data).sum()
            rows.append(airport)

    df = pd.DataFrame(rows)
    df['prob_infected'] = df.catchment_cases / df.catchment_pop
    df = df.query('catchment_pop > 0')
    print(df.sort_values('prob_infected')[['prob_infected', 'name']])


# Import libraries flask for web server backend processing, random and json python libraries
from flask import Flask, render_template, jsonify, request, json
from multiprocessing import Process, freeze_support, set_start_method
from random import randint
import json
import multiprocessing as mp
import subprocess

app = Flask(__name__) # Initialise flask app instance

# Function to execute flask application
def runFlask():
    app.run(host='0.0.0.0', port=8080, debug=False, use_reloader=True)

# App route to direct to root, renders index.html (main page)
@app.route("/")
def index():
    return render_template('index.html')

# App route "get data" to receive simulation data for parking spot status updates
@app.route('/_get_data/', methods=['GET', 'POST'])
def get_data():
    tempCoorStore = []

    with open('gdf.json', 'r') as f: # Opens pre-process json file which stores coordinates of parking spaces
        coordinateJson = json.load(f)
        coordinateJsonConverted = json.loads(coordinateJson) # Converts into python dictionary for manipulation
        for items in coordinateJsonConverted['features']:
            tempCoorStore.append(items) # Add dictionary items into list for processing
        for i in range(len(tempCoorStore)):
            tempCoorStore[i]['properties'] = {'avail': str(randint(0,1))} # Modifies key "properties" to dictionary
                                                                          # with "avail" and stringified integer as
                                                                          # value
    f.close()
    finalDict = {"type": "FeatureCollection", "features": tempCoorStore} # Constructs final dictionary fit for
                                                                         # jsonification
    return jsonify({'coordinates': json.dumps(finalDict)}) # Returns jsonified dictionary into frontend JavaScript

if __name__ == "__main__":
    p1 = Process(target=runFlask) # Defines process to start Flask for multiprocessing
    p1.start() # Executes Flask


import sqlite3
from sqlite3 import Error

def create_connection(file):
    conn = None
    try:
        conn = sqlite3.connect(file)
        print(sqlite3.version)
    except Error as e:
        print(e)
    return conn

def create_table(conn, sql):
    try:
        c = conn.cursor()
        c.execute(sql)
    except Error as e:
        print(e)

def create_database(file):
    sql_offers_table = """ CREATE TABLE IF NOT EXISTS goods (
                            id integer PRIMARY KEY,
                            modified_time text NOT NULL,
                            categoryId text NOT NULL,
                            vendor text NOT NULL,
                            name text NOT NULL,
                            description text NOT NULL,
                            param text NOT NULL,
                            picture text NOT NULL,
                            oldprice text NOT NULL,
                            price text NOT NULL,
                            url text NOT NULL
                        ); """

    conn = create_connection(file)
    if conn is not None:
        create_table(conn, sql_offers_table)
    else:
        print('Ошибка! Не получается создать подключение к Базе данных!')

def add_offer(conn, row):
    sql = """INSERT INTO goods(modified_time, categoryId, vendor, name, description, param, picture, oldprice, price,
            url) VALUES(?,?,?,?,?,?,?,?,?,?) """
    cur = conn.cursor()
    cur.execute(sql, row)
    conn.commit()
    return cur.lastrowid

if __name__ == '__main__':
    # database = r'database.db'
    # create_database(database)
    pass


#!/usr/bin/python3

import os
import re
import cv2
import time
import pytesseract
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor

TEMP_DIR = 'temp'
EVIDENCE_DIR = 'evidence'
os.makedirs(TEMP_DIR, exist_ok=True)
os.makedirs(EVIDENCE_DIR, exist_ok=True)

def extract_frames(filepath, file, interval):
    # check if the file is a video file
    _, ext = os.path.splitext(filepath)
    if ext not in ['.mp4', '.avi', '.mkv']:
        return

    # use OpenCV to extract frames
    try:
        cap = cv2.VideoCapture(filepath)
    except:
        print("Failed to load video")

    try:
        fps = int(cap.get(cv2.CAP_PROP_FPS))
    except:
        print("Failed to load video (fps)")

    frame_count = 0
    while True:
        ret, frame = cap.read()

        if not ret:
            break

        if frame_count > 1200: # 10 Minutes every 30 seconds snapshot
           cap.release()
           break

        if frame_count % (fps * interval) == 0:
            try:
                temp_frame = os.path.join(TEMP_DIR, f'inspect_{file}_{frame_count}.jpg')
                temp_content = os.path.join(TEMP_DIR, f'content_{file}_{frame_count}.dat')
                cv2.imwrite(temp_frame, frame)
            except:
                print("Failed to analyze video")
            try:
                data = pytesseract.image_to_string(temp_frame, config='-c tessedit_char_whitelist=abcdefghijklmnopqrstuvwxyz01234567890.://@%?=&,;_- ') # Tune OCR

                #
                # ML model goes here basic
                #

                keywords = {
                            'social': ['snapchat','facebook','tiktok','instagram','framatalk','discord','youtube','omegle'],
                            'gambling':['draftkings','gambling'],
                            'pornography':['porn','pornhub','xhamster','xvidoes','sex'],
                            'games':['roblox','steam','minecraf','now.gg','fortnite'],
                            'crypto':['miner','crypto'],
                            'hacking':['masscan'],
                            'proxies':['proxy','ocks5','browserling','homeworkistrash','vpn'],
                            'software':['downloads','setup','.exe','.zip','github','anonfiles'],
                            'accounts':['password','inbox'],
                            'illegal':['torrent','checker']
                        }

                keystore = []
                for key, values in keywords.items():
                    for value in values:
                        keystore.append([key, value])

                for category,keyword in keystore:

                    fraud = re.search(keyword, data)

                    if fraud is not None:
                        evidence_frame = os.path.join(EVIDENCE_DIR, f'{category}_{file}_{frame_count}.jpg')
                        evidence_content = os.path.join(EVIDENCE_DIR, f'{category}_{file}_{frame_count}.dat')
                        cv2.imwrite(evidence_frame, frame)
                        with open(evidence_content, 'a') as f:
                            f.write(data)

            except:
                print("OCR Reading Error")
        frame_count += 1
    cap.release()

def list_files(directory):
    directory = os.path.abspath(directory)
    for root, dirs, files in os.walk(directory):
        for file in files:
            path = os.path.join(root, file)
            print(path)
            executor = ThreadPoolExecutor(max_workers=4)
            executor.submit(extract_frames,path,file,30)
            time.sleep(1.5) # prevents runway threads tune

list_files("videos/") # Change this to location of video files in mp4, avi, mkv formats


from fastapi import FastAPI
from DoItList.Users.router import router as users_router
from DoItList.Items.router import router as items_router

app = FastAPI(title='ToDo App')
from fastapi.middleware.cors import CORSMiddleware

app.include_router(users_router)
app.include_router(items_router)

origins = ['http://localhost:3000',
           'http://localhost',
           'http://localhost:8080']
app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=['*'],
    allow_headers=['*'],
)


import setuptools
import subprocess
import os

with open("README.md", "r") as fh:
    long_description = fh.read()

requirements = [
    'osmium',
    'pyparsing',
    'pyyaml',
    'shapely',
    'requests',
    'landez'
]

setuptools.setup(
    name="osm-export-tool",
    version="0.1.0",
    author="Maxar Technologies",
    author_email="foundationgeoint-packaging@maxar.com",
    description="Convert OpenStreetMap data into GIS and mobile mapping file formats.",
    license="BSD-3-Clause",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/radiant-maxar/osm-export-tool-python",
    packages=setuptools.find_packages(),
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: BSD License",
        "Operating System :: OS Independent",
    ],
    scripts=['bin/osm-export-tool'],
    install_requires = requirements,
    requires_python='>=3.0',
    package_data={'osm_export_tool':['mappings/*.yml']}
)


from flask import Flask, request
import requests
import os
import dataset

app = Flask(__name__)

rai_node_address = 'http://%s:%s' % ('127.0.0.1', '7076')
database_location = 'sqlite:///%s' % 'hashdata.db'

def get_work(hash):
     get_work = '{ "action" : "work_generate", "hash" : "%s", "use_peers": "true" }' % hash
     r = requests.post(rai_node_address, data = get_work)
     print(r.text)
     resulting_work = r.json()
     return resulting_work['work'].lower()

def get_account_from_hash(hash):
     get_account = '{ "action" : "block_account", "hash" : "%s"}' % hash
     r = requests.post(rai_node_address, data = get_account)
     resulting_data = r.json()
     if 'account' in resulting_data:
         account = resulting_data['account']
         return account
     else:
         return 'Error'

@app.route('/work', methods=['POST'])
def generate_work():
     hash = request.form['hash']
     print(hash)
     account_db = dataset.connect(database_location)
     account_table = account_db['account']
     account = get_account_from_hash(hash)
     if account == 'Error':
         work_output = get_work(hash)
         return_json = '{"work" :"%s"}' % work_output
         return return_json

     precache_work = account_table.find_one(account=account)
     if precache_work != None :
         print('Found cached work value')
         work_output = precache_work['work']
     else:
         work_output = get_work(hash)
         account_table.insert(dict(account=account, hash=hash, work=work_output))

     return_json = '{"work" :"%s"}' % work_output

     return return_json


from __future__ import annotations
from dataclasses import dataclass
from dataclasses import field
from urllib.parse import urlsplit
from typing import Any, Optional, TypedDict
from lib.types import Headers
from entities.payload_file import PayloadFile, PayloadFileSerialised
from entities.model import Model

from constants import CALCULATED_TEXT
from lib.types import Headers
from lib.input_parsing.text_wrapper import parse_text, parse_text_with_payload_values
from copy import deepcopy
from mitmproxy.common_types import ProxyRequest

def escape_quotes(value: str) -> str:
    return value.replace('"', r'\"')

class FuzzFormData(TypedDict):
    payload_files: list[PayloadFileSerialised]
    fuzz_type: str
    delay_type: str
    delay_secs: Optional[str]
    delay_secs_min: Optional[str]
    delay_secs_max: Optional[str]

class FormData(TypedDict):
    method: str
    url: str
    headers: Headers
    content: str
    fuzz_data: Optional[FuzzFormData]

@dataclass(kw_only=True)
class HttpRequest(Model):
    # Columns
    id: int = field(init=False, default=0)
    created_at: int = field(init=False, default=0)

    http_version: str
    headers: Headers
    content: Optional[str] = None
    trailers: Optional[str] = None
    timestamp_start: Optional[float] = None
    timestamp_end: Optional[float] = None
    host: str
    port: int
    method: str
    scheme: str
    authority: Optional[str] = None
    path: str
    form_data: FormData

    # Relations

    meta = {
        "relationship_keys": [],
        "json_columns": ["headers", "form_data"],
        "do_not_save_keys": [],
    }

    FUZZ_TYPE_LABELS = ['One To One', 'Cartesian Product']
    FUZZ_TYPE_KEYS = ['one_to_one', 'cartesian']

    DELAY_TYPE_LABELS = ['Disabled', 'Fixed Time Delay', 'Random Time Delay']
    DELAY_TYPE_KEYS = ['disabled', 'fixed', 'random']

    # This is how requests are received from the proxy
    @classmethod
    def from_state(cls, state: ProxyRequest) -> HttpRequest:
        request = HttpRequest(
            http_version = state['http_version'],
            headers = dict(state['headers']),
            content = state['content'],
            trailers = state['trailers'],
            timestamp_start = state['timestamp_start'],
            timestamp_end = state['timestamp_end'],
            host = state['host'],
            port = state['port'],
            method = state['method'],
            scheme = state['scheme'],
            authority = state['authority'],
            path = state['path'],
            form_data = {
                'method': state['method'],
                'url': '',
                'headers': dict(state['headers']),
                'content': state['content'],
                'fuzz_data': None
            },
        )

        request.form_data['url'] = request.get_url()

        return request

    # TODO: Use a TypedDict instead of Any
    # TODO: Make this work
    def get_state(self) -> dict[str, Any]:
        # attributes = self.serialize()
        # attributes['headers'] = json.loads(attributes['headers'])
        return {}

    @classmethod
    def build_blank_for_editor(cls):
        return HttpRequest(
            http_version = 'HTTP/1.1',
            headers = {},
            host = '',
            port = 80,
            method = 'GET',
            scheme = 'http',
            path = '',
            content = '',
            form_data = {'method': 'GET', 'url': 'http://', 'headers': {}, 'content': '', 'fuzz_data': None},
        )

    @classmethod
    def build_blank_placeholder(cls):
        return HttpRequest(
            http_version = 'HTTP/1.1',
            headers = {},
            host = '',
            port = 80,
            method = 'GET',
            scheme = 'http',
            path = '',
            content = '',
            form_data = {'method': 'GET', 'url': 'http://', 'headers': {}, 'content': '', 'fuzz_data': None},
        )

    def set_headers(self, headers: Headers) -> None:
        self.headers = headers

    def get_headers(self) -> Headers:
        return self.headers

    def get_header_line_no_http_version(self) -> str:
        return f'{self.method} {self.path}'

    def get_header_line(self) -> str:
        return f'{self.method} {self.path} {self.http_version}'

    def get_url(self) -> str:
        if self.port not in [80, 443, None]:
            port = ':' + str(self.port)
        else:
            port = ''

        return f'{self.scheme}://{self.host}{port}{self.path}'

    def duplicate(self) -> HttpRequest:
        return HttpRequest(
            http_version = self.http_version,
            headers = self.headers,
            content = self.content,
            trailers = self.trailers,
            host = self.host,
            port = self.port,
            method = self.method,
            scheme = self.scheme,
            authority = self.authority,
            path = self.path,
            form_data = self.form_data or self.generate_form_data(),
        )

    def modify(self, modified_method: str, modified_path: str, modified_headers: Headers, modified_content: str):
        self.method = modified_method
        self.path = modified_path
        self.headers = modified_headers
        self.content = modified_content

        if modified_headers.get('Host'):
            host, port = self.__get_host_and_port(modified_headers['Host'])
            self.host = host
            if port:
                self.port = port
        elif modified_headers.get('host'):
            # TODO: DRY up this duplication
            host, port = self.__get_host_and_port(modified_headers['Host'])
            self.host = host
            if port:
                self.port = port

    def overwrite_calculated_headers(self) -> None:
        calc_text = CALCULATED_TEXT
        headers = self.get_headers()

        if headers is None:
            return

        if headers.get('Host'):
            headers['Host'] = calc_text

        if headers.get('Content-Length'):
            headers['Content-Length'] = calc_text

        self.set_headers(headers)

    def apply_payload_values(self, payload_values: dict[str, str]) -> None:
        method = parse_text_with_payload_values(str(self.form_data['method']), payload_values)
        url = parse_text_with_payload_values(str(self.form_data['url']), payload_values)
        content = parse_text_with_payload_values(str(self.form_data['content']), payload_values)

        # TODO:
        headers = self.form_data['headers']

        self.set_form_data({
            'method': method,
            'url': url,
            'headers': headers,
            'content': content,
            'fuzz_data': None
        })
        return

    def reset_form_data(self) -> None:
        self.set_form_data(self.form_data)

    def set_form_data(self, form_data: FormData) -> None:
        self.form_data = form_data

        # 1. Set method
        self.method = str(self.form_data['method'])

        # 2. Set URL related attributes
        parsed_url = parse_text(str(self.form_data['url']))
        url_data = urlsplit(parsed_url)
        if url_data.hostname:
            self.host = url_data.hostname

        if url_data.port:
            self.port = url_data.port
        else:
            if url_data.scheme == "https":
                self.port = 443
            else:
                self.port = 80

        self.scheme = url_data.scheme

        if url_data.query == '':
            self.path = url_data.path
        else:
            self.path = url_data.path + '?' + url_data.query

        # 3. Set content
        self.content = parse_text(str(form_data['content']))

        # 4. Set headers
        parsed_headers = deepcopy(form_data['headers'])
        for key, value in form_data['headers'].items():
            parsed_headers[key] = parse_text(value)

        self.set_headers(parsed_headers)

    def generate_form_data(self) -> FormData:
        headers = self.get_headers() or {}
        content = self.content or ''

        return {
            'method': self.method,
            'url': self.get_url(),
            'headers': headers,
            'content': content,
            'fuzz_data': None
        }

    def fuzz_data(self) -> Optional[FuzzFormData]:
        return self.form_data.get('fuzz_data')

    def payload_files(self) -> list[PayloadFile]:
        fuzz_data = self.fuzz_data()

        if fuzz_data is None:
            return []

        return [PayloadFile.from_serialised(p) for p in fuzz_data['payload_files']]

    def payload_keys(self) -> list[str]:
        fuzz_data = self.fuzz_data()

        if fuzz_data is None:
            return []

        return [p['key'] for p in fuzz_data['payload_files']]

    def get_curl_command(self) -> str:
        headers_dict = self.get_headers()
        headers = ""

        if headers_dict is not None:
            headers = ['"{0}: {1}"'.format(k, escape_quotes(v)) for k, v in headers_dict.items() if v != CALCULATED_TEXT]
            headers = " -H ".join(headers)

        content = getattr(self, 'content', None)
        if content is None or content == "":
            return f"curl -X {self.method} -H {headers} \"{self.get_url()}\""
        else:
            return f"curl -X {self.method} -H {headers} -d {content}' {self.get_url()}"

    def __eq__(self, other) -> bool:
        return (
            self.method == other.method and
            self.host == other.host and
            self.port == other.port and
            self.path == other.path and
            self.headers == other.headers and
            self.content == other.content
        )

    def __get_host_and_port(self, host: str) -> tuple[str, Optional[int]]:
        host_arr = host.split(':')
        if len(host_arr) == 2:
            return (host_arr[0], int(host_arr[1]))
        else:
            return [host_arr[0], None] # type: ignore


'''
2018 KAKAO BLIND RECRUITMENT
[1차] 프렌즈4블록
https://programmers.co.kr/learn/courses/30/lessons/17679
'''

def solution(m, n, board):
    while True:
        success = []        # 2x2 성공한 블록
        for i in range(1, m):
            for j in range(1, n):
                if board[i][j] == "_":      # 비어있는 블록은 패스
                    continue

                if board[i-1][j-1] == board[i-1][j] == board[i][j-1] == board[i][j]:        # 2x2 조건 성립한 블록
                    success.append((i, j))

        if not success:     # 2x2에 해당하는 블록이 없으면 종료
            break

        for i, j in success:        # 2x2에 해당하는 블록은 _처리
            board[i-1] = board[i-1][:j-1] + "__" + board[i-1][j+1:]
            board[i] = board[i][:j-1] + "__" + board[i][j+1:]

        for i in range(m - 1, 0, -1):       # 밑 블록부터 탐색하여 _인 블록을 만나면 상단 블록에서 _이 아닌 블록을 가져와서 교환
            for j in range(n):
                if board[i][j] == "_":
                    for _i in range(i - 1, -1, -1):
                        if board[_i][j] != "_":
                            board[i] = board[i][:j] + board[_i][j] + board[i][j+1:]
                            board[_i] = board[_i][:j] + "_" + board[_i][j+1:]
                            break

    return sum([x.count("_") for x in board])       # _블럭들의 개수를 구하여 반환

'''
여담

2x2 블록 찾아서 _처리하는 것은 쉬웠는데,
빈공간을 윗 블록들이 채우는 로직을 생각한다고 시간을 좀 썼다.
그래도 1시간 내에 풀이한 거 같아 다행이다.
for문 중첩이 많아 소모 시간이 클 것이라 예상했는데, 생각보다 그렇게 많은 시간이 걸리진 않더라.
'''


import gymnasium as gym
import numpy as np

import torch
import torch.nn as nn
import torch.nn.functional as F

from tqdm import tqdm, trange

device = 'cpu'

def make_env(env_id, seed, idx, run_name):
	def thunk():
		env = gym.make(env_id)
		env.action_space.seed(seed)
		env.observation_space.seed(seed)
		return env
	return thunk()

hidden_dim = 1024

def layer_init(layer, std=np.sqrt(2), bias_const=0.0):
	nn.init.orthogonal_(layer.weight, std)
	nn.init.constant_(layer.bias, bias_const)
	return layer

class Agent(nn.Module):
	def __init__(self, envs):
		super().__init__()

		self.critic = nn.Sequential(
			layer_init(nn.Linear(np.array(envs.single_observation_space.shape).prod(), hidden_dim)),
			nn.Tanh(),
			layer_init(nn.Linear(hidden_dim, hidden_dim)),
			nn.Tanh(),
			layer_init(nn.Linear(hidden_dim, 1), std=1.0)
		)

		self.actor_mean = nn.Sequential(
			layer_init(nn.Linear(np.array(envs.single_observation_space.shape).prod(), hidden_dim)),
			nn.Tanh(),
			layer_init(nn.Linear(hidden_dim, hidden_dim)),
			nn.Tanh(),
			layer_init(nn.Linear(hidden_dim, np.array(envs.single_action_space.shape).prod()), std=0.01)
		)
		self.actor_logstd = nn.Parameter(torch.zeros(1, np.prod(envs.single_action_space.shape)))

	def get_value(self, x):
		return self.critic(x)

	def get_action_and_value(self, x, pre_action=None):
		pass

envs = gym.vector.SyncVectorEnv([
    lambda: make_env('Pusher-v4', 0+i, i, 'the first') for i in range(4)
])

agent = Agent(envs).to(device)
# print(agent.load_state_dict(torch.load('models/pusher_ppo/final.pth')))
agent.load_state_dict(torch.load('models/pusher_ppo/1408.pth'))

env = gym.make('Pusher-v4', render_mode='human')
state, info = env.reset()

loop = tqdm()
while True:
# for _ in range(512):
	with torch.no_grad():
		action = agent.actor_mean(torch.from_numpy(state).float().unsqueeze(0).to(device))
		# action = agent.actor_mean(action)
		# action, _, _, _ = agent.get_action_and_value(torch.from_numpy(state).float().to(device))
	state, reward, terminated, truncated, _ = env.step(action[0].cpu().numpy())

	loop.set_postfix(reward=reward)
	if terminated or truncated:
		state, info = env.reset()


'''

Example 3

Input:
7 5
....$
.....
.....
.DDD.
.....
.....
@....

Output:
1

Example 4

Input:
7 5
@....
.....
.....
D...D
.....
.....
....$

Output:
2

Example 5

Input:
7 5
@....
.....
.....
.DDD.
.....
.....
....$

Output:
1
'''

#Solution:

import sys
import queue
import heapq

def eat(x):
    if x[0] == 'D':
        return True
    if x[0] == '@':
        return False
    if x[0] == '$':
        return False
    return None

def trans(x):
    if x[0] == 'D':
        return 'D'
    if x[0] == '@':
        return '@'
    if x[0] == '$':
        return '$'
    if x[0] == '.':
        return '.'
    return None

def distance(x, y):
    return abs(x[0] - y[0]) + abs(x[1] - y[1])

def get_eat(grid, start, end, monster_pos):
    if eat(start) == False and eat(end) == None:
        return 0
    if start == end:
        return 0
    if eat(start) == None and eat(end) == False:
        return distance(start, end)
    if eat(start) == eat(end):
        return 0
    if eat(start) == True and eat(end) == False:
        visited = set()
        q = []
        heapq.heappush(q, (0, end))
        while q:
            curr = heapq.heappop(q)
            if curr[1] in visited:
                continue
            visited.add(curr[1])
            if eat(grid[curr[1][0]][curr[1][1]]) == True and distance(curr[1], end) <= 2:
                visited.clear()
                return distance(curr[1], start)
            for n in neighbors(grid, curr[1]):
                if eat(grid[n[0]][n[1]]) == False:
                    heapq.heappush(q, (curr[0] + 1, n))
        return 1000000
    if eat(start) == False and eat(end) == True:
        visited = set()
        q = []
        heapq.heappush(q, (0, start))
        while q:
            curr = heapq.heappop(q)
            if curr[1] in visited:
                continue
            visited.add(curr[1])
            if eat(grid[curr[1][0]][curr[1][1]]) == True and distance(curr[1], end) <= 2:
                visited.clear()
                return distance(curr[1], start)
            for n in neighbors(grid, curr[1]):
                if eat(grid[n[0]][n[1]]) == False:
                    heapq.heappush(q, (curr[0] + 1, n))
        return 1000000

def neighbors(grid, curr):
    h = len(grid)
    w = len(grid[0])
    ret = []
    if curr[0] > 0:
        ret.append((curr[0] - 1, curr[1]))
        if curr[1] > 0:
            ret.append((curr[0] - 1, curr[1] - 1))
        if curr[1] < w - 1:
            ret.append((curr[0] - 1, curr[1] + 1))
    if curr[0] < h - 1:
        ret.append((curr[0] + 1, curr[1]))
        if curr[1] > 0:
            ret.append((curr[0] + 1, curr[1] - 1))
        if curr[1] < w - 1:
            ret.append((curr[0] + 1, curr[1] + 1))
    if curr[1] > 0:
        ret.append((curr[0], curr[1] - 1))
    if curr[1] < w - 1:
        ret.append((curr[0], curr[1] + 1))
    return ret

def compute(grid):
    h = len(grid)
    w = len(grid[0])
    monster_pos = []
    start = None
    end = None
    for i in range(h):
        for j in range(w):
            if trans(grid[i][j]) == '@':
                start = (i, j)
            if trans(grid[i][j]) == '$':
                end = (i, j)
            if trans(grid[i][j]) == 'D':
                monster_pos.append((i, j))
    return get_eat(grid, start, end, monster_pos)

def main():
    sys.stdin = open('input.txt', 'r')
    sys.stdout = open('output.txt', 'w')
    arr = list(map(int, input().split()))
    grid = []
    h = arr[0]
    w = arr[1]
    for i in range(h):
        grid.append(list(input()))
    print(compute(grid))

if __name__ == '__main__':
    main()


import argparse
import os
import logging
import sys
from typing import List, Generator

# list of file extensions, all lowercase, without leading "."
DEFAULT_SIDECAR_EXTENSIONS = [
    "xmp",
    "pp3"
]

def get_args(args):
    '''Configures command line parser and returns parsed parameters'''
    parser = argparse.ArgumentParser(
        description="""Find sidecar files (like XMP) which don't
        have the associated base file anymore""")
    parser.add_argument(
        "base_folder",
        help="Base folder to recursively search for orphaned sidecar files")
    parser.add_argument(
        "-d", "--debug", dest="debuglevel", action="store_const",
        const=logging.DEBUG, default=logging.INFO,
        help="Enables verbose/debug output")
    parser.add_argument(
        "-e", "--extensions", type=lambda x: x.lower().split(","),
        default=DEFAULT_SIDECAR_EXTENSIONS,
        help="Comma seperated list of sidecar file extensions. Defaults to \"{}\"".format(
            ",".join(DEFAULT_SIDECAR_EXTENSIONS)))
    return parser.parse_args(args)

def get_orphaned_files(files_in_folder: List[str], sidecar_extensions: List[str]) -> List[str]:
    '''Returns orphaned files from a specific folder

    In the following example both abc.def.xyz and abc.def are considered as sidecar
    files for abc.def:

    abc.def
    abc.def.xyz
    abc.xyz

    Arguments:

    files_in_folder: List of filenames in folders
    sidecar_extensions: List of sidecar file extensions

    Returns:

    List of orphaned files
    '''

    result = []
    sidecar_files = [] # list of sidecar only files
    orig_files_base = [] # original file names, no sidecar files

    for entry in files_in_folder:
        base, ext = os.path.splitext(entry)
        if ext.lower().lstrip(".") in sidecar_extensions:
            sidecar_files.append(entry)
        else:
            # Match for abc.def
            orig_files_base.append(entry)
            # Match for abc.xyz
            orig_files_base.append(base)

    for entry in sidecar_files:
        base, ext = os.path.splitext(entry)
        if not base in orig_files_base:
            result.append(entry)

    return result

def find_files(base_folder: str, sidecar_extensions) -> Generator[str, None, None]:
    '''Find sidecar files

    Arguments:

    base_folder: The base folder where to start the recursive search
    sidecar_extensions: List of sidecar file extensions

    Returns a list of duplicates (yield)
    '''

    for root, _, files in os.walk(base_folder, followlinks=True):
        if files:
            logging.debug("Checking %s with %d files", root, len(files))
            orphaned_files = get_orphaned_files(files, sidecar_extensions)
            if orphaned_files:
                for file_ in orphaned_files:
                    yield os.path.join(root, file_)

def main():
    '''main function, called when script file is executed directly'''
    args = get_args(sys.argv[1:])
    logging.basicConfig(format="%(message)s", level=args.debuglevel)
    for entry in find_files(args.base_folder, args.extensions):
        print(entry)

if __name__ == "__main__":
    main()


from socket import *
import time
import struct

# funk from the web to get chat from user
def getch():
    import termios
    import sys
    import tty

    def _getch():
        fd = sys.stdin.fileno()
        old_settings = termios.tcgetattr(fd)
        try:
            tty.setraw(fd)
            ch = sys.stdin.read(1)
        finally:
            termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
        return ch

    return _getch()

class Client:
    def __init__(self, ip):
        self.ip = ip
        self.buffer_size = 1024

    def udp_part(self, counter):
        server_address_port = (self.ip, 13217)
        print("Client started, listening for offer requests...")

        # init the UDP socket
        udp_client_socket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)
        udp_client_socket.setsockopt(SOL_SOCKET, SO_REUSEPORT, 1)
        udp_client_socket.setsockopt(SOL_SOCKET, SO_BROADCAST, 1)
        udp_client_socket.settimeout(10)

        # wait for some connection
        to_continue = True
        try:
            udp_client_socket.bind(server_address_port)
            receive_message, server_address = udp_client_socket.recvfrom(self.buffer_size)
            print("Received offer from {} attempting to connect...".format(server_address[0]))
            tcp_ip = server_address[0]
            receive_message = struct.unpack('IBH', receive_message)
        except:
            time.sleep(0.5)
            if counter > 5:
                time.sleep(5)
            self.udp_part(counter + 1)
            to_continue = False

        if to_continue:
            # organize recv data
            magic_cookie = hex(receive_message[0])
            tcp_port = hex(receive_message[2])
            tcp_port = int(tcp_port, 16)
            if magic_cookie == '0xfeedbeef':
                self.tcp_part(tcp_port, tcp_ip, counter)

    def tcp_part(self, tcp_port, tcp_ip, counter):
        # init the TCP socket
        server_address = (tcp_ip, int(tcp_port))
        tcp_client_socket = socket(AF_INET, SOCK_STREAM)
        try:
            to_continue = True
            tcp_client_socket.connect(server_address)
        except:
            tcp_client_socket.close()
            time.sleep(0.5)
            if counter > 5:
                time.sleep(5)
            self.udp_part(counter + 1)
            to_continue = False

        if to_continue:
            # send team name
            tcp_client_socket.send(str.encode("FSM\n"))
            message = ""
            try:
                tcp_client_socket.settimeout(10)
                message = tcp_client_socket.recv(self.buffer_size)
            except:
                tcp_client_socket.close()
                time.sleep(0.5)
                if counter > 5:
                    time.sleep(5)
                self.udp_part(counter + 1)
                to_continue = False

            if to_continue:
                message = message.decode("utf-8")
                print(message)

                # start the game !!!
                future = time.time() + 10
                while future > time.time():
                    try:
                        tcp_client_socket.send(str.encode(getch()))
                    except:
                        continue

                # recv end message
                try:
                    tcp_client_socket.settimeout(10)
                    message = tcp_client_socket.recv(self.buffer_size)
                    message = message.decode("utf-8")
                    print(message)
                except:
                    time.sleep(0.5)
                self.udp_part(0)

eth1 = "172.1.0.24"
eth2 = "172.99.0.24"
broadcast_ip = "<broadcast>"
c1 = Client("")
c1.udp_part(0)


###############################################################
################# https://www.fardanesh.ir ####################
###############################################################

from openpyxl import Workbook
from openpyxl.drawing.image import Image,PILImage

wb=Workbook()
ws1=wb.active

if PILImage:
    img=Image('/app/lecture09-tables-images/python-logo.png')
    img.width=200
    img.height=200
    ws1.add_image(img,'B5')

wb.save(filename='lecture09-tables-images/imageAdded.xlsx')


from jobs.jobIngestionProviders import DataIngestion
from pyspark.sql import SparkSession

def get_spark_session():
    spark = (
        SparkSession.builder
        .config("spark.yarn.dist.files", "/home/nan/gecko/code/companies-ingestion/prospect_ingestion/out/prospect_ingestion.pex")
        .config("spark.jars.packages",  "org.apache.hudi:hudi-spark3.1-bundle_2.12:0.12.0")
        .getOrCreate()
    )
    return spark

def main():
    spark = get_spark_session()
    DataIngestion(spark).run()

if __name__ == "__main__":
    main()


#!/usr/bin/env python3

import sys
sys.path.append('./Anafora/')

import os, configparser
import anafora

def main(xml_dir, text_dir, xml_regex, context_size):
  """Main Driver"""

  for sub_dir, text_name, file_names in anafora.walk(xml_dir, xml_regex):

    xml_path = os.path.join(xml_dir, sub_dir, file_names[0])
    ref_data = anafora.AnaforaData.from_file(xml_path)

    text_path = os.path.join(text_dir, text_name)
    text = open(text_path).read()

    for data in ref_data.annotations.select_type('EVENT'):

      start, end = data.spans[0]
      context = text[start-context_size:end+context_size].replace('\n', '')
      event = text[start:end]
      dtr = data.properties['DocTimeRel']

      left = text[start-context_size:start]
      right = text[end:end+context_size]
      tagged = left + ' [ES] ' + event + ' [EE] ' + right
      print('{}|{}|{}'.format(dtr, event, context))
      print(tagged)
      print()

if __name__ == "__main__":

  cfg = configparser.ConfigParser()
  cfg.read(sys.argv[1])
  base = os.environ['DATA_ROOT']

  xml_dir = os.path.join(base, cfg.get('data', 'train_xml'))
  text_dir = os.path.join(base, cfg.get('data', 'train_text'))
  xml_regex = cfg.get('data', 'xml_regex')
  context_size = 10

  main(xml_dir, text_dir, xml_regex, context_size)


from collections import deque

bullet_price = int(input())
barrel_size = int(input())
bullets = list(map(int, input().split()))
locks = list(map(int, input().split()))
intelligence_value = int(input())
fired_bullets = 0
barrel = deque()

for _ in range(barrel_size):
    if bullets:
        barrel.append(bullets.pop())

while (bullets or barrel) and locks:
    current_bullet = barrel.popleft()
    fired_bullets += 1
    if current_bullet <= locks[0]:
        locks.pop(0)
        print('Bang!')
    else:
        print('Ping!')

    if not barrel:
        if bullets:
            print('Reloading!')
            for _ in range(barrel_size):
                if bullets:
                    barrel.append(bullets.pop())
                else:
                    break
        else:
            break

if locks:
    print(f'Couldn\'t get through. Locks left: {len(locks)}')
else:
    print(f'{len(bullets) + len(barrel)} bullets left. Earned ${intelligence_value - fired_bullets * bullet_price}')

1. Lists as stacks and queues/EXCERCISE/09_key_revolver.py


from api.factories import ExampleFactory

from django.core.management.base import BaseCommand

class Command(BaseCommand):
    help = "Seeds the database."

    def add_arguments(self, parser):
        parser.add_argument(
            "--example",
            default=200,
            type=int,
            help="The number of fake examples to create.",
        )

    def handle(self, *args, **options):
        print("Populating database...")
        for _ in range(options["example"]):
            ExampleFactory()
        print("Done.")


import sys
N = int(input())
li = list(map(int, sys.stdin.readline().split()))
length = [0] * N
for i in range(N):
    length[i] = 1
    for j in range(0,i):
        if li[j] < li[i]:
            length[i] = max(length[j]+1, length[i])
print(length)


# -*- coding:utf-8 -*-
import xadmin
from .models import Acitity,AcitityComments,Application

class AcitityAdmin(object):
    list_display = ('id','city','brand', 'serie','name','start_date','end_date','customer_qty')
    list_display_links = ('name',)

    search_fields = ['name','city','brand','serie']
    list_filter =('city','brand','serie')
    readonly_fields = ('id',)

xadmin.site.register(Acitity, AcitityAdmin)

class AcitityCommentsAdmin(object):
    list_display = ('id', 'activity','user', 'DOS','serie')
    list_display_links = ('id',)

    search_fields = ['activity','user']
    list_filter =('activity','serie')
    readonly_fields = ('id',)

xadmin.site.register(AcitityComments, AcitityCommentsAdmin)

class ApplicationAdmin(object):
    list_display = ('id', 'activity','name', 'phone','way','target_brand','target_serie','target_version')
    list_display_links = ('name',)

    search_fields = ['activity','name','phone'],
    list_filter =('activity','way','target_brand','target_serie','target_color','target_time','way_check_car')
    readonly_fields = ('id',)
xadmin.site.register(Application, ApplicationAdmin)


import io
import json
from api.sc2_extract import sc2_extract
from http.server import BaseHTTPRequestHandler
from urllib.request import urlopen

class Encoder(json.JSONEncoder):
  def default(self, obj):
    if isinstance(obj, bytes):
      try:
        return obj.decode('utf-8')
      except:
        return ''
    return json.JSONEncoder.default(self, obj)

class handler(BaseHTTPRequestHandler):
  def do_GET(self):
    if (self.path[:8] == '/api/gg-'): url = f'http://ggtracker.com/matches/{self.path[8:]}/replay'
    if (self.path[:8] == '/api/rs-'): url = f'http://sc2replaystats.com/download/{self.path[8:]}'
    if (self.path[:8] == '/api/st-'): url = f'http://lotv.spawningtool.com/{self.path[8:]}/download/'
    raw = sc2_extract(io.BytesIO(urlopen(url).read()))
    del raw['gameevents'] # HACK: Need to find a wait around zeit 5m content size limit
    self.send_response(200)
    self.send_header('Access-Control-Allow-Origin', '*')
    self.send_header('Access-Control-Request-Method', '*')
    self.send_header('Access-Control-Allow-Methods', 'OPTIONS, GET')
    self.send_header('Access-Control-Allow-Headers', '*')
    self.send_header('Cache-Control', 's-maxage=31536000')
    self.send_header('Content-Type', 'application/json')
    self.end_headers()
    self.wfile.write(json.dumps(raw, cls=Encoder, indent=2, sort_keys=True).encode())
    return


# ---
# jupyter:
#   jupytext:
#     formats: ipynb,py:light
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.13.1
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

from matplotlib import pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
sns.set()

def read_heartbeats_log(filename: str):
    df = pd.read_csv(filename, delimiter=" ",
                     skipinitialspace=True, index_col="Beat")
    df = df.groupby("Tag").mean()
    return df

# # Benchmark AVX2 (without POET )

# +
import numpy as np
from sklearn.linear_model import LinearRegression
from utils import system_cpu_freqs

def rate_power(folder,freq):
    df_hb = read_heartbeats_log(f"../experiments/{folder}/heartbeat_{freq}.log")
    power = df_hb["Global_Power"].mean()
    rate =  df_hb["Global_Rate"].mean()
    return rate,power

freqs = np.array(list(system_cpu_freqs()))

without_avx2_32 = np.array([rate_power("without_avx2_32",f) for f in freqs])
without_avx2_64 = np.array([rate_power("without_avx2_64",f) for f in freqs])

with_avx2_32 = np.array([rate_power("with_avx2_32",f) for f in freqs])/without_avx2_32
with_avx2_64 = np.array([rate_power("with_avx2_64",f) for f in freqs])/without_avx2_64

reg32 = LinearRegression().fit(np.array(freqs).reshape(-1,1), with_avx2_32[:,0])
reg64 = LinearRegression().fit(np.array(freqs).reshape(-1,1), with_avx2_64[:,0])

plt.figure(figsize=(9,5))
plt.plot(freqs,with_avx2_32[:,0],"--o",label="AVX2 32-bit",color="C0")
plt.plot(freqs,with_avx2_64[:,0],"--o",label="AVX2 64-bit",color="C1")

plt.plot(freqs,np.array(freqs)*reg32.coef_+reg32.intercept_,"-",color="C0")
plt.text(1.8e6,1.11,"avg. "+str(with_avx2_32[:,0].mean().round(3)),ha="center",color="C0")

plt.plot(freqs,np.array(freqs)*reg64.coef_+reg64.intercept_,"-",color="C1")
plt.text(2.3e6,1.055,"avg. "+str(with_avx2_64[:,0].mean().round(3)),ha="center",color="C1")
plt.xticks(freqs)
plt.ylabel("Speedup")
plt.xlabel("CPU Frequency (GHz)")
plt.legend()
plt.savefig("../figures/avx2_speedup.pdf",dpi=300,bbox_inches='tight')

# +
reg32 = LinearRegression().fit(np.array(freqs).reshape(-1,1), with_avx2_32[:,1])
reg64 = LinearRegression().fit(np.array(freqs).reshape(-1,1), with_avx2_64[:,1])

plt.figure(figsize=(9,5))
plt.plot(freqs,with_avx2_32[:,1],"o",label="AVX2 32-bit",color="C0")
plt.plot(freqs,with_avx2_64[:,1],"o",label="AVX2 64-bit",color="C1")

plt.plot(freqs,np.array(freqs)*reg32.coef_+reg32.intercept_,"-",color="C0")
plt.text(2.3e6,1.055,"avg. "+str(with_avx2_32[:,1].mean().round(3)),ha="center",color="C0")

plt.plot(freqs,np.array(freqs)*reg64.coef_+reg64.intercept_,"-",color="C1")
plt.text(1.8e6,1.0,"avg. "+str(with_avx2_64[:,1].mean().round(3)),ha="center",color="C1")
plt.xticks(freqs)
plt.ylabel("Powerup")
plt.xlabel("CPU Frequency (GHz)")
plt.legend()
plt.savefig("../figures/avx2_powerup.pdf",dpi=300,bbox_inches='tight')
# -

# # Benchmark with POET

# +
df_poet = pd.read_csv("../experiments/poet_no_avx2_32/poet.log", delimiter=" ",
                      skipinitialspace=True).groupby("HB_NUM").first()
df_heartbeats = read_heartbeats_log("../experiments/poet_no_avx2_32/heartbeat.log")

df_poet_avx2 = pd.read_csv("../experiments/poet_avx2_32/poet.log", delimiter=" ",
                      skipinitialspace=True).groupby("HB_NUM").first()
df_heartbeats_avx2 = read_heartbeats_log("../experiments/poet_avx2_32/heartbeat.log")

# +
df_avx2 = df_poet_avx2.join(df_heartbeats_avx2)
df_avx2["Timestamp"] = pd.to_datetime(df_avx2["Timestamp"] - df_avx2["Timestamp"].iloc[0]).iloc[5:]

df = df_poet.join(df_heartbeats)
df["Timestamp"] = pd.to_datetime(df["Timestamp"] - df["Timestamp"].iloc[0]).iloc[5:]

# +
fig, ax1 = plt.subplots(figsize=(20, 5))

ax1.plot([df["Timestamp"].min(),df["Timestamp"].max()],[180]*2,label="Target Rate",color="green")

ax1.plot(df["Timestamp"], df["Global_Rate"], label="Rate", color="C0")
ax1.plot(df_avx2["Timestamp"], df_avx2["Global_Rate"], label="Rate (AVX2)", color="C2")

ax1.tick_params(axis='y', labelcolor="C0")
ax1.set_ylabel('Rate', color="C0")
ax1.set_xlabel('Time', color="black")

ax2 = ax1.twinx()
ax2.plot(df["Timestamp"], df["Global_Power"], label="Power", color="C1")
ax2.plot(df_avx2["Timestamp"], df_avx2["Global_Power"], label="Power (AVX2)", color="C3")

ax2.tick_params(axis='y', labelcolor="C1")
ax2.set_ylabel('Power (Watt)', color="C1")
ax2.grid(False)

step=20
num_threads = [1,2,3,4,5,6,5,4,3,2,1]
areas = [[t,[i*step,(i+1)*step]] for i,t in enumerate(num_threads)]

for cores, a in areas:
    x_min = pd.Timestamp(0)+pd.Timedelta(a[0], unit="sec")
    x_max = pd.Timestamp(0)+pd.Timedelta(a[1], unit="sec")
    ax1.axvspan(x_min, x_max,
                color=f"C{int(cores)+2}", alpha=0.3)
    y = sum(ax1.get_ylim())/2
    ax1.text(pd.Timestamp(0)+pd.Timedelta((a[0]+a[1])/2, unit="sec"),
             y, f"{cores} thread(s)",
             color=f"C{int(cores)+2}", horizontalalignment="center")

fig.legend(ncol=5,loc="upper center")
fig.savefig("../figures/poet_benchmark.pdf",dpi=300,bbox_inches='tight')
# -

# # Datasets Benchmark

# +
from os import path
import glob

ds_n = []

for fn in glob.glob("../experiments/datasets_32/heartbeat_*.log"):
    parts = path.basename(fn).split(".")[0].split("_")[2:5]
    dc = dict(zip(["Size","Distribution","Ratio"],parts))
    df_h = read_heartbeats_log(fn)
    dc["Operations / Second"] = df_h["Instant_Rate"].mean() * 10000
    watt = df_h["Instant_Power"].mean()
    delta =pd.to_datetime(df_h.iloc[-1]["Timestamp"])-pd.to_datetime(df_h.iloc[0]["Timestamp"])

    joules = watt*delta.total_seconds()
    dc["Operations/Joule"] = int(dc["Operations / Second"]/joules)
    dc["Operations / Second"] = int(dc['Operations / Second'])
    ds_n.append(dc)

df_ds = pd.DataFrame(ds_n)
df_ds
# -

df_ds_n = df_ds.sort_values(["Size","Distribution","Ratio"],ascending=False).reset_index()
df_ds_n.index = list(range(1,len(df_ds)+1))
del df_ds_n["index"]
df_ds_n

print(df_ds_n.to_latex())


import os
from typing import Any, Dict

import pytest
import yaml

from swh.core.api import RPCServerApp
from swh.core.config import load_from_envvar
from swh.search.api import server
from swh.search.api.server import load_and_check_config, make_app_from_configfile

def teardown_function():
    # Ensure there is no configuration loaded from a previous test
    server.api_cfg = None

def _write_config_file(tmp_path, monkeypatch, content):
    conf_path = os.path.join(str(tmp_path), "search.yml")
    with open(conf_path, "w") as f:
        f.write(yaml.dump(content))
    monkeypatch.setenv("SWH_CONFIG_FILENAME", conf_path)
    return conf_path

@pytest.fixture
def swh_search_server_config_without_indexes() -> Dict[str, Any]:
    return {
        "search": {
            "cls": "elasticsearch",
            "hosts": ["es1"],
        }
    }

@pytest.fixture
def swh_search_server_config_with_indexes(
    swh_search_server_config_without_indexes,
) -> Dict[str, Any]:
    return {
        "search": {
            **{"indexes": {"origin": {"index": "test"}}},
            **swh_search_server_config_without_indexes["search"],
        }
    }

@pytest.fixture
def swh_search_config_without_indexes(
    monkeypatch, swh_search_server_config_without_indexes, tmp_path
):
    return _write_config_file(
        tmp_path, monkeypatch, swh_search_server_config_without_indexes
    )

@pytest.fixture
def swh_search_config_with_indexes(
    monkeypatch, swh_search_server_config_with_indexes, tmp_path
):
    return _write_config_file(
        tmp_path, monkeypatch, swh_search_server_config_with_indexes
    )

def prepare_config_file(tmpdir, config_dict: Dict, name: str = "config.yml") -> str:
    """Prepare configuration file in `$tmpdir/name` with content `content`.

    Args:
        tmpdir (LocalPath): root directory
        content: Content of the file either as string or as a dict.
                            If a dict, converts the dict into a yaml string.
        name: configuration filename

    Returns
        path of the configuration file prepared.

    """
    config_path = tmpdir / name
    config_path.write_text(yaml.dump(config_dict), encoding="utf-8")
    # pytest on python3.5 does not support LocalPath manipulation, so
    # convert path to string
    return str(config_path)

@pytest.mark.parametrize("config_file", [None, ""])
def test_load_and_check_config_no_configuration(config_file):
    """Inexistent configuration files raises"""
    with pytest.raises(EnvironmentError, match="Configuration file must be defined"):
        load_and_check_config(config_file)

def test_load_and_check_config_inexistent_file():
    config_path = "/some/inexistent/config.yml"
    expected_error = f"Configuration file {config_path} does not exist"
    with pytest.raises(EnvironmentError, match=expected_error):
        load_and_check_config(config_path)

def test_load_and_check_config_wrong_configuration(tmpdir):
    """Wrong configuration raises"""
    config_path = prepare_config_file(tmpdir, {"something": "useless"})
    with pytest.raises(KeyError, match="Missing 'search' configuration"):
        load_and_check_config(config_path)

def test_load_and_check_config_local_config_fine(
    swh_search_server_config_with_indexes, tmpdir
):
    """'local' complete configuration is fine"""
    config_path = prepare_config_file(tmpdir, swh_search_server_config_with_indexes)
    cfg = load_and_check_config(config_path)
    assert cfg == swh_search_server_config_with_indexes

def test_server_make_app_from_config_file_without_indexes(
    swh_search_config_without_indexes,
):
    app = make_app_from_configfile()
    expected_cfg = load_from_envvar()

    assert app is not None
    assert isinstance(app, RPCServerApp)
    assert app.config["search"] == expected_cfg["search"]

    app2 = make_app_from_configfile()
    assert app is app2

def test_server_make_app_from_config_file_with_indexes(
    swh_search_config_with_indexes,
):
    app = make_app_from_configfile()
    expected_cfg = load_from_envvar()
    assert app is not None
    assert isinstance(app, RPCServerApp)
    assert app.config["search"] == expected_cfg["search"]

    app2 = make_app_from_configfile()
    assert app is app2

def test_server_first_call_initialize_elasticsearch(
    swh_search_config_with_indexes, mocker
):
    """Test the initialize method is called during the first and first only
    request to the server"""
    mock = mocker.patch("swh.search.elasticsearch.ElasticSearch.initialize")

    app = make_app_from_configfile()
    app.config["TESTING"] = True
    tc = app.test_client()

    tc.get("/")
    assert mock.call_count == 1

    tc.get("/")
    assert mock.call_count == 1


# accpet the string from user

name = raw_input("Enter the name\n")
date = raw_input("Enter date\n")
print("Good morning," + name)

#
sample='''Hello <name>

You are selected!!!!

Date : <Date>
'''

print(sample.replace("<name>",name).replace("<Date>",date))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Mar 19 10:13:38 2019

@author: parkerglenn
"""
from sklearn.decomposition import PCA
import matplotlib.pylab as plt
import seaborn as sns
from sklearn.decomposition import TruncatedSVD
from matplotlib import pyplot
import mpld3
from mpld3 import display
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.cluster import KMeans
import pylab

###############################################################################
##################VISUALIZATION################################################
###############################################################################

#Date Distribution
out = pd.cut(df["joined"], bins = [20000000,20020000,20040000,20060000,20080000,20100000,20120000,20140000,20160000,20180000], include_lowest = True)
ax = out.value_counts(sort = False).plot.bar(rot = 0, color = "b", figsize = (20,10))
ax.set_ylim(bottom = 0, top = 450)
ax.set_xticklabels(x for x in ["2000 to 2002","2002 to 2004","2004 to 2006","2006 to 2008","2008 to 2010","2010 to 2012","2012 to 2014","2014 to 2016","2016 to 2018"])
plt.xlabel("Date Range", fontsize = 18)
plt.ylabel("Frequency", fontsize = 18)
plt.title("Date Distribution", fontsize = 25)
for i in ax.patches:
    ax.text(i.get_x() + .10, 5, str(i.get_height()), fontsize = 20, color = "black")

#Distance Heatmap
dist = 1 - cosine_similarity(matrix)

cmap = pyplot.cm.cubehelix
dimensions = (20,20)
fig, ax = pyplot.subplots(figsize=dimensions)
sns.heatmap(dist, vmin = 0, vmax = 1, cmap = cmap).set_title("Tfidf Distances Between Articles", fontsize = 15)

"""
Notice hot spot around the 625:630 line.
Those article titles:
['Dem Debate Blogging #1',
 'Dem Debate Blogging #2',
 'Dem Debate Blogging #3',
 'Dem Debate Blogging #4',
 'Dem Debate Blogging #5',
 'Dem Debate Blogging #6']

Around 90:160: large circle of relatively similar articles.
Reason: Iowa Caucuses.
"""

#Tfidf Matrix, in 2D SVD scatterplot
svd = TruncatedSVD(n_components=2).fit(matrix)
data2D = svd.transform(matrix)
plt.title("Truncated SVD, 2 Components")
colors = rng.rand(1000)
plt.scatter(data2D[:,0], data2D[:,1], marker = "o", c = colors, cmap = "YlGnBu", s = 10)

######With clusters assigned as colors########
data2D = svd.transform(matrix)
kmeans = KMeans(n_clusters = 520)
kmeans.fit(data2D)
y_kmeans = kmeans.predict(data2D)
y_pred = kmeans.labels_.tolist()

success(kmeans,y_pred,matrix)

articles = {"title": titles, "date": new_df["date"], "cluster": y_pred, "content": new_df["content"], "event": events[:1000]}
frame = pd.DataFrame(articles, index = [y_pred] , columns = ['title', 'date', 'cluster', 'content', "event"])
frame['cluster'].value_counts()

"""Creates scalable points for cluster centers found within y_true.
The size of the center is dependent on how many events occur withing that specific cluster. """

centers = kmeans.cluster_centers_
fig, ax = plt.subplots(figsize = (14,8))
np.random.seed(0)
threshold = 4
for cluster, center in enumerate(centers):
    cluster+=1
    # Only maps cluster if it has more than "threshold" events in it
    if cluster in y_true and len(frame.loc[cluster]["event"].values.tolist()) > threshold:
        #Gets event name that the cluster center represents
        event = events[y_true.index(cluster)]
        #s scaled based on number of events in cluster
        ax.plot(center[0], center[1], markersize = float(len(frame.loc[cluster]["event"].values.tolist())),  marker = "o");
        plt.annotate(event, (center[0],center[1]))
ax.set_title('Cluster Centers with Predominant Event', size=14)
plt.show()

mpld3.show(fig)

# mpld3.save_html(fig, "Cluster_Centers.html")

"""Zoomed out plot with colors"""
plt.title("Truncated SVD with Colored Clusters")
plt.scatter(data2D[:, 0], data2D[:, 1], c=y_kmeans, cmap = "tab20", s=30)
"""Example plot, zoomed in to visualize cluster centers"""

fig, ax = plt.subplots(figsize = (14,8))
np.random.seed(0)
ax.plot(data2D[:, 0], data2D[:, 1],
        'or', ms=10, alpha=0.2)
ax.set_title('Truncated SVD with Cluster Assignments', size=14)
ax.grid(color='lightgray', alpha=0.7)
for i, txt in enumerate(events):
    print(i)
    plt.annotate(txt + ", " + str(y_pred[i]), (data2D[:, 0][i], data2D[:, 1][i]))
mpld3.show(fig)

centers = kmeans.cluster_centers_
plt.scatter(centers[:, 0], centers[:, 1], c='black', s=30, alpha=0.5, marker = 'x');
pyplot.axis(ymax = 0, ymin =-.25 , xmax = .2, xmin = .14)
plt.title("Truncated SVD with Cluster Centers")
pyplot.axis(ymax = 0, ymin =-.25 , xmax = .2, xmin = .14)
plt.scatter(data2D[:, 0], data2D[:, 1], c=y_kmeans, cmap = "tab20", s=30)

######Interactive ScatterPlot with SVD######
svd = TruncatedSVD(n_components=2).fit(matrix)
data2D = svd.transform(matrix)

fig, ax = plt.subplots(figsize = (14,8))
np.random.seed(0)
ax.plot(data2D[:, 0], data2D[:, 1],
        'or', ms=10, alpha=0.2)
ax.set_title('Truncated SVD with Cluster Assignments', size=14)
ax.grid(color='lightgray', alpha=0.7)
for i, txt in enumerate(events):
    print(i)
    plt.annotate(txt + ", " + str(y_pred[i]), (data2D[:, 0][i], data2D[:, 1][i]))
mpld3.show(fig)
#mpld3.save_html(fig, "Truncated_SVD_D3.html")

#####Interactive ScatterPlot with Dense Matrix and PCA######
"""Probably not the method to use. SVD seems better since it takes the sparse matrix "tfidf_matrix" directly."""
x = tfidf_matrix.todense()
coords = PCA(n_components=2).fit_transform(x)
fig, ax = plt.subplots(figsize = (14,8))

np.random.seed(0)
ax.plot(coords[:, 0], coords[:, 1],
        'or', ms=10, alpha=0.2)
ax.set_title('PCA with Cluster Assignments', size=14)
ax.grid(color='lightgray', alpha=0.7)
for i, txt in enumerate(events):
    plt.annotate(txt, (coords[:, 0][i], coords[:, 1][i]))
mpld3.show(fig)

"""Dendogram Making"""
fig = pylab.figure(figsize=(100,70))
children = hac.children_
distance = np.arange(children.shape[0])
no_of_observations = np.arange(2, children.shape[0]+2)
linkage_matrix = np.column_stack([children, distance, no_of_observations]).astype(float)
dendrogram(linkage_matrix, labels = (events), truncate_mode = "level", leaf_font_size = 8)
fig.show

"""ScatterPlot"""
coords = PCA(n_components=2).fit_transform(dense_matrix)
fig, ax = plt.subplots(figsize = (14,8))
np.random.seed(0)
ax.plot(coords[:, 0], coords[:, 1],
        'or', ms=10, alpha=0.2)
ax.set_title('Truncated SVD with Cluster Assignments', size=14)
ax.grid(color='lightgray', alpha=0.7)
for i, txt in enumerate(events):
    print(i)
    plt.annotate(txt + ", " + str(y_pred[i]), (coords[:, 0][i], coords[:, 1][i]))
mpld3.show(fig)

#############################################################
################Hyperparameter Testing#######################
#############################################################

hyper_params = pd.read_csv("/Users/parkerglenn/Desktop/DataScience/Article_Clustering/csv/Hyperparam_testing/Hyper_Params.csv")
hyper_params = hyper_params.reset_index()

f_scores = [x for x in hyper_params["f_score"]]
s_scores = [x for x in hyper_params["s_score"]]
person_rate = [x for x in hyper_params["person_rate"]]
ents_rate = [x for x in hyper_params["ents_rate"]]

"""F1 Score distribution across hyperparameters
Best F1 Score: ents_rate = 1.63, person_rate = 2.57, f_score = .927 on 412 articles. BUT s_score is a measly .0788.
"""
f_data = pd.DataFrame({"Person Rate": [round(x,2) for x in person_rate], "Entity Rate":[round(x,2) for x in ents_rate], "Z": hyper_params.f_score})
f_data_pivoted = f_data.pivot("Person Rate","Entity Rate","Z")
ax1 = sns.heatmap(f_data_pivoted, cmap = plt.cm.hot)
cbar = ax1.collections[0].colorbar
cbar.set_label('F1 Score', labelpad=15)
ax1.invert_yaxis()
plt.show()

"""Silhouette Score distribution across hyperparameters
More direct correlation here than in F1 Score. Most notably, as the rates increase, so does S Score. This is due to
the fact that the rating of words in articles becomes more radical; articles that are different from each other, i.e. share no
entities, are now much more distant than before. Even more specifically, when the Entity Rate and Person Rate are dissimilar, the
S Score is the highest. Again, the same radical rating phenomena: By limiting the amount of weighted features, those that are weighted
make the article more of an outlier than before."""

s_data = pd.DataFrame({"Person Weighting": [round(x,2) for x in person_rate], "Non-Person Weighting":[round(x,2) for x in ents_rate], "Z": hyper_params.s_score})
s_data_pivoted = s_data.pivot("Person Weighting","Non-Person Weighting","Z")
ax2 = sns.heatmap(s_data_pivoted, cmap = plt.cm.hot)
cbar = ax2.collections[0].colorbar
cbar.set_label('Silhouette Score', labelpad=15)
ax2.invert_yaxis()
plt.show()

"""Cumulative Total
Highest: ents_rate = 6.368, person_rate = 2.263, f_score = 0.922, s_score = 0.093

"""
sf_data = pd.DataFrame({"Person Weighting": [round(x,2) for x in person_rate], "Entity Weighting":[round(x,2) for x in ents_rate], "Z": hyper_params.s_score + hyper_params.f_score})
sf_data_pivoted = sf_data.pivot("Person Weighting","Entity Weighting","Z")
ax3 = sns.heatmap(sf_data_pivoted, cmap = plt.cm.hot)
cbar = ax3.collections[0].colorbar
cbar.set_label('Composite Score', labelpad=15)
ax3.invert_yaxis()
plt.show()

"""Little thing to find best cumulative score and rates that achieved it."""
hightot = 0
for tup in enumerate(hyper_params["f_score"]):
    tot = tup[1] + hyper_params.loc[tup[0]]["s_score"]
    if hightot < tot:
        hightot = tot
        best_scores = (hyper_params.loc[tup[0]]["ents_rate"],hyper_params.loc[tup[0]]["person_rate"])
best_scores


import os
import pickle

import nltk
from nltk import word_tokenize
from nltk.util import ngrams

def find_accuracy(filepath, filepath2):
    file = open(os.path.join(os.getcwd(), filepath), 'r')
    file2 = open(os.path.join(os.getcwd(), filepath2), 'r')

    total = 0
    correct = 0
    incorrect = list()

    while True:
        line1 = file.readline()
        line2 = file2.readline()
        total += 1

        if not line1:
            break

        if line1 == line2:
            correct += 1
        else:
            incorrect.append(line1.split()[0])

    file.close()
    file2.close()

    acc = correct / total

    print("accuracy %.2f" % acc)
    print("incorrect line numbers", incorrect)

def find_most_likely(filepath, u_eng, b_eng, u_fre, b_fre, u_ita, b_ita):
     with open(os.path.join(os.getcwd(), filepath), 'r') as file:

        count = 1

        with open('results.txt', 'w') as f:
            for sentence in file:
                most_likely = get_lang(sentence, u_eng, b_eng, u_fre, b_fre, u_ita, b_ita)
                f.write(str(count) + " " + most_likely + "\n")
                count += 1

def get_lang(sentence, u_eng, b_eng, u_fre, b_fre, u_ita, b_ita):
     eng = compute_prob(sentence, u_eng, b_eng, len(u_eng), len(b_eng))
     fre = compute_prob(sentence, u_fre, b_fre, len(u_fre), len(b_fre))
     ita = compute_prob(sentence, u_ita, b_ita, len(u_ita), len(b_ita))

     biggest = max(eng, fre, ita)

     if eng == biggest:
        return "English"
     elif fre == biggest:
        return "French"
     else:
         return "Italian"

def compute_prob(text, unigram_dict, bigram_dict, N, V):
    # N is the number of tokens in the training data
    # V is the vocabulary size in the training data (unique tokens)

    unigrams_test = word_tokenize(text)
    bigrams_test = list(ngrams(unigrams_test, 2))

    p_laplace = 1  # calculate p using Laplace smoothing

    for bigram in bigrams_test:
        n = bigram_dict[bigram] if bigram in bigram_dict else 0
        d = unigram_dict[bigram[0]] if bigram[0] in unigram_dict else 0

        p_laplace = p_laplace * ((n + 1) / (d + V))

    return p_laplace

if __name__ == '__main__':

        # read the pickle files
        unidict_eng = pickle.load(open('unidict_eng.p', 'rb'))  # read binary
        bidict_eng = pickle.load(open('bidict_eng.p', 'rb'))  # read binary

        # read the pickle files
        unidict_fre = pickle.load(open('unidict_fre.p', 'rb'))  # read binary
        bidict_fre = pickle.load(open('bidict_fre.p', 'rb'))  # read binary

        # read the pickle files
        unidict_ita = pickle.load(open('unidict_ita.p', 'rb'))  # read binary
        bidict_ita = pickle.load(open('bidict_ita.p', 'rb'))  # read binary

        find_most_likely("data/LangId.test", unidict_eng, bidict_eng, unidict_fre, bidict_fre, unidict_ita, bidict_ita)
        find_accuracy("data/LangId.sol", "results.txt")


import os
import json
from cowait.tasks import TaskDefinition
from cowait.engine import get_cluster_provider, env_unpack, \
    ENV_TASK_CLUSTER, ENV_TASK_DEFINITION, ENV_GZIP_ENABLED

def env_get(key):
    if key not in os.environ:
        raise ValueError(f'Missing required environment variable {key}')
    value = os.environ[key]
    if os.getenv(ENV_GZIP_ENABLED, '0') == '1':
        return env_unpack(value)
    return json.loads(value)

def env_get_cluster_provider():
    clusterdef = env_get(ENV_TASK_CLUSTER)
    return get_cluster_provider(**clusterdef)

def env_get_task_definition():
    taskdef = env_get(ENV_TASK_DEFINITION)
    return TaskDefinition.deserialize(taskdef)


from math import sqrt
from os import system

SMALLEST_PRIME = 2

def main():

    UI.welcome()

    n = int(input('    Enter a number: '))

    UI.computing()

    primes = compute_primes_in(n)
    factors = compute_factors_in(n)

    print(primes)
    print(factors)

    print(f'        {n} is {("even" if is_even(n) else "odd")}')
    print(f'        {n} is {("" if is_prime(n) else "not ")}a prime')
    UI.line()

def compute_primes_in(_n):
    primes = set()
    for i in range(3, _n, 2):
        if (is_prime(i)): primes.add(i)
    return primes

def compute_factors_in(_n):
    factors = list()
    _f = SMALLEST_PRIME

    while _n > 1:
        if not (_n % _f):
            _n /= _f
            factors.append(_f)
        else:
            _f = get_next_prime(_f)

    return factors

def get_next_prime(_n):
    _n += 1
    while (not is_prime(_n)): _n += 1
    return _n

def is_even(n):
    return not (n % 2)

def is_prime(n):
    if is_even(n): return False

    for i in range(3, int(sqrt(n)), 2):
        if not (n % i): return False

    return True

class UI:
    def h1(s):
        print(f'        {s}')

    def h2(s):
        print(f'        {s}')

    def welcome():
        system('clear')

        UI.line()
        UI.h1('PRIMES')
        UI.line()

    def computing():
        UI.line()
        UI.h2('Calculating...')
        UI.line()

    def line():
        print('-' * 32)

if __name__ == '__main__':
    main()