#!/usr/bin/env python3
###################################################################
#
#   CSSE1001/7030 - Assignment 1
#
#   Student Username: s4442951
#
#   Student Name: Mounir Hedna
#
###################################################################

###################################################################
#
# The following is support code. DO NOT CHANGE.

from a1_support import *

# End of support code
################################################################

# Write your code here

def interact():
    # Add your code for interact here
    """Interact will continuously loop until and ask the user for input until the game is over"""

    COMMANDS = "n s e w r b p q ?"
    """To add new commands, add the new character to the commands string making
    sure there is a space in between each character"""

    file = input("Maze File: ").strip()
    maze = load_maze(file)
    current_position = START_POSITION
    moves = []

    game_over = False
    while(not game_over):

        print()
        print_maze(maze, current_position)
        print()

        try:
            command = input("Command: ").strip()
        except EOFError:
            pass

        legal_moves = get_legal_directions(maze, current_position)

        if command not in COMMANDS or command == "":
            print("Invalid command: " + command)

        elif command in legal_moves:
            pos = move(maze, current_position, command)
            if pos[1] in BAD_POKEMON:
                print(LOSE_TEXT.format(POKEMON.get(pos[1])))
                game_over = True
            elif pos[1] in GOOD_POKEMON:
                print(WIN_TEXT.format(POKEMON.get(pos[1])))
                game_over = True

            else:
                moves.append(current_position)
                current_position = pos[0]

        elif command == "r":
            current_position = START_POSITION
            moves = []

        elif command == "b":
            if len(moves) == 0:
                print("You cannot go back from the beginning.")

            else:
                current_position = moves.pop()

        elif command == "?":
            print(HELP_TEXT)

        elif command == "p":
            p = ""
            count = 0

            for m in legal_moves:
                if count == len(legal_moves)-1:
                    p += m
                else:
                    p += m + ", "
                    count += 1

            print("Possible directions: " + p)

        elif command == "q":
            try:
                quit_game = input("Are you sure you want to quit? [y] or n: ").strip()

            except EOFError:
                pass

            if quit_game == "y":
                game_over = True

        else:
            print("You can't go in that direction.")

def get_position_in_direction(position, direction):
    """Takes a position (coordinate) and a direction from "n,e,s,w" and returns the
       new coordinate.

       get_position_in_direction((int, int), str) -> (int, int)
       """
    dP = DIRECTION_DELTAS.get(direction)
    x = position[1] + dP[1]
    y = position[0] + dP[0]

    return (y, x)

def print_maze(maze, position):
    """Takes a maze and a position (coordinate) and prints the maze with position marked
    with an 'A'.

    print_maze(str, (int, int)) -> None
    """
    rows = maze.split("\n")
    new_row = list(rows[position[0]])
    new_row[position[1]] = PLAYER
    rows[position[0]] = "".join(new_row)

    for row in rows:
        print(row)

def move(maze, position, direction):
    """Takes in a maze, a position and a direction. Returns the position after moving
       in the given direction and the value at that position.

       move(str, (int, int), str) -> ((int, int), str)
       """
    position2 = get_position_in_direction(position, direction)
    rows = maze.split("\n")
    row = rows[position2[0]]
    value = row[position2[1]]

    if value == WALL:
        return (position, WALL)
    else:
        return (position2, value)

def get_legal_directions(maze, position):
     """Takes a maze and a position coordinate and returns all legal directions
        you are able to move in.

        get_legal_directions(str, (int, int)) -> list<str>
        """
     available_directions = []
     for d in DIRECTIONS:
         m = move(maze, position, d)
         if m[1] != WALL:
             available_directions += d
     return available_directions

##################################################
# !!!!!! Do not change (or add to) the code below !!!!!
#
# This code will run the interact function if
# you use Run -> Run Module  (F5)
# Because of this we have supplied a "stub" definition
# for interact above so that you won't get an undefined
# error when you are writing and testing your other functions.
# When you are ready please change the definition of interact above.
###################################################

if __name__ == '__main__':
    interact()


import cv2
from glob import glob

imgs1 = glob('../dataset/GetChu_1-75/*')
imgs2 = glob('../dataset/GetChu_76-118/*')
imgs = imgs1+imgs2
print(len(imgs))

from tqdm import tqdm

def detect_face(imgs):
    face_cascade = cv2.CascadeClassifier('lbpcascade_animeface.xml')
    for img in tqdm(imgs):
        img_temp = cv2.imread(img)
        gray = cv2.cvtColor(img_temp, cv2.COLOR_BGR2GRAY)
        gray = cv2.equalizeHist(gray)
        faces = face_cascade.detectMultiScale(gray, scaleFactor=1.1, minNeighbors=5, minSize=(24, 24))
        for i, face in enumerate(faces):
            (stx, sty, w, h) = face
            stx -= (w * 0.3)
            sty -= (h * 0.3)
            w *= 1.5
            h *= 1.45
            stx = max(0, stx)
            sty = max(0, sty)
            name = img.split('/')[-1].split('.')[0] + '_face' + str(i) + '.jpg'
            img_new = img_temp[int(sty):int(sty + h), int(stx):int(stx + w), :]
            cv2.imwrite('../dataset/GetChu_aligned/' + name, img_new)

from multiprocessing import Pool
# 分割path
split_num=8
temp_len=len(imgs)//split_num
base_paths=[]
for i in range(split_num):
    if i != split_num-1:
        base_paths.append(imgs[i*temp_len:(i+1)*temp_len])
    else:
        base_paths.append(imgs[i*temp_len:])

pool=Pool(split_num)
result=[]
for i in base_paths:
    result.append(pool.apply_async(detect_face, kwds={'imgs':i}))
pool.close()
pool.join()


class solution:
    def findCeil(self, lst: list[int], x: int):
        l, r = 0, len(lst) - 1
        if x > lst[r]:
            return - 1

        while l <= r:
            m = (l + r)//2
            if x > lst[m]:
                l = m + 1
            else:
                r = m - 1
        return lst[l]

x = solution()
print(x.findCeil([1, 2, 8, 10, 10, 12, 19], 4))


import tensorflow as tf

from datasets import pascalvoc_to_tfrecords

FLAGS = tf.app.flags.FLAGS

tf.app.flags.DEFINE_string(
    'dataset_name', 'pascalvoc',
    'The name of the dataset to convert.')
tf.app.flags.DEFINE_string(
    'dataset_dir', None,
    'Directory where the original dataset is stored.')
tf.app.flags.DEFINE_string(
    'output_name', 'pascalvoc',
    'Basename used for TFRecords output files.')
tf.app.flags.DEFINE_string(
    'output_dir', './',
    'Output directory where to store TFRecords files.')

def main(_):
    if not FLAGS.dataset_dir:
        raise ValueError('You must supply the dataset directory with --dataset_dir')
    print('Dataset directory:', FLAGS.dataset_dir)
    print('Output directory:', FLAGS.output_dir)

    if FLAGS.dataset_name == 'pascalvoc':
        pascalvoc_to_tfrecords.run(FLAGS.dataset_dir, FLAGS.output_dir, FLAGS.output_name)
    else:
        raise ValueError('Dataset [%s] was not recognized.' % FLAGS.dataset_name)

if __name__ == '__main__':
    tf.app.run()


#!/usr/bin/env python

from CrombieTools.PlotTools.PlotStack import *
from CrombieTools.LoadConfig import cuts
from array import array
import sys, os

SetupFromEnv()

#plotter.SetDebug(True)

plotter.AddDataFile('wscale_Data.root')
plotter.SetStackLineWidth(2)
plotter.SetCMSLabel('Preliminary')
plotter.SetTreeName('events')
plotter.SetAllHistName('htotal')
plotter.SetLegendLocation(plotter.kUpper, plotter.kRight, 0.25, 0.5)
plotter.SetEventsPer(1.0)
plotter.SetMinLegendFrac(0.0)
plotter.SetIgnoreInLinear(0.005)
plotter.SetOthersColor(922)
plotter.SetFontSize(0.03)
plotter.SetAxisTitleOffset(1.2)
plotter.SetRatioMinMax(0,2)

plotter.SetForceTop('W-matched (tt)')

def SetupArgs():
    return [
        ['npv', 50, 0, 50, 'NPV', 'Events/1.0'],
        ['fatjetMaxBTag', 25, 0, 1.25, 'Max subjet b-tag (csv)', 'Events/1.0'],
#        ['jet_ht', 40, 0, 2000, 'Jet HT (jet p_{T} > 30 GeV)', 'Events/1.0'],
#        ['fatjetDPhiMet', 25, 0, 5, '#Delta #phi from MET', 'Events/1.0'],
        ['met', 20, 0, 400, 'MET [GeV]', 'Events/1.0'],
        ['mt', 20, 0, 200, 'm_{T}', 'Events/1.0'],
        ['n_tightlep', 4, 0, 4, 'Num Tight Leptons', 'Events/1.0'],
#        ['n_mediumlep', 4, 0, 4, 'Num Medium Leptons', 'Events/1.0'],
        ['n_jetsTot', 10, 0, 10, 'Num Jets (p_{T} > 30 GeV)', 'Events/1.0'],
        ['n_jetsNotFat', 10, 0, 10, 'Num Jets (p_{T} > 30 GeV)', 'Events/1.0'],
        ['n_bjetsLoose', 6, 0, 6, 'Num Loose b Jets', 'Events/1.0'],
        ['n_bjetsMedium', 6, 0, 6, 'Num Medium b Jets', 'Events/1.0'],
        ['n_bjetsTight', 6, 0, 6, 'Num Tight b Jets', 'Events/1.0'],
#        ['fatjetPt', 25, 100, 600, 'p_{T} [GeV]', 'Events/1.0'],
        ['fatjetPt', 25, 250, 600, 'p_{T} [GeV]', 'Events/1.0'],
        ['fatjetPtSmearedCentral', 25, 100, 600, 'p_{T} [GeV]', 'Events/1.0'],
        ['fatjetPtSmearedUp', 25, 100, 600, 'p_{T} [GeV]', 'Events/1.0'],
        ['fatjetPtSmearedDown', 25, 100, 600, 'p_{T} [GeV]', 'Events/1.0'],
        ['fatjetMass', 25, 0, 250, 'Fat Jet Mass [GeV]', 'Events/1.0'],
        ['fatjetEta', 50, -5.0, 5.0, 'Fat Jet Eta', 'Events/1.0'],
        [{'cut_lines': [65.0, 105.0]}, 'fatjetPrunedML2L3', 25, 0, 250, 'Fat Jet Pruned Mass [GeV]', 'Events/1.0'],
        [{'data_expr': 'fatjetPrunedML2L3', 'cut_lines': [65.0, 105.0]}, 'fatjetPrunedML2L3SmearedCentral', 25, 0, 250, 'Fat Jet Pruned Mass [GeV]', 'Events/1.0'],
        [{'data_expr': 'fatjetPrunedML2L3', 'cut_lines': [65.0, 105.0]}, 'fatjetPrunedML2L3SmearedUp', 25, 0, 250, 'Fat Jet Pruned Mass [GeV]', 'Events/1.0'],
        [{'data_expr': 'fatjetPrunedML2L3', 'cut_lines': [65.0, 105.0]}, 'fatjetPrunedML2L3SmearedDown', 25, 0, 250, 'Fat Jet Pruned Mass [GeV]', 'Events/1.0'],
        [{'cut_lines': [0.6]}, 'fatjettau21', 25, 0, 1.5, '#tau_{2}/#tau_{1}', 'Events/1.0'],
#        ['fatjettau32', 25, 0, 1.5, '#tau_{2}/#tau_{1}', 'Events/1.0'],
        ['fatjetDRLooseB', 25, 0, 5, '#Delta R from Loose b', 'Events/1.0'],
#        ['fatjetDRMediumB', 25, 0, 5, '#Delta R from Medium b', 'Events/1.0'],
#        ['fatjetDRTightB', 25, 0, 5, '#Delta R from Tight b', 'Events/1.0'],
        ['fatjetDPhiLep1', 25, 0, 5, '#Delta #phi from Lepton', 'Events/1.0'],
        ['thirdFatMass', 25, 0, 15, 'Pruned Mass of Other Fat Jets', 'Events/1.0'],
        ]

def RunPlots(categories):
    cutList = ['nocut', 'nocut_nsmalljets', 'full', 'full_highpt', 'full_massp_tau21']
    for move in range(8):
        cutList.append('full_' + str(move * 0.1))

    MakePlots(categories, cutList, SetupArgs())

    plotter.SetDefaultWeight(cuts.cut('semilep', 'full_nopt'))
    plotter.SetMCWeight('(fatjetPtSmearedUp > 250) * (' + cuts.defaultMCWeight + ')')
    plotter.SetDataWeight('(fatjetPt > 250)')
    plotter.SetDataExpression('fatjetPrunedML2L3')
    plotter.SetDefaultExpr('fatjetPrunedML2L3SmearedUp')
    plotter.MakeCanvas('smearedup_mass', 25, 0, 250, 'Fat Jet Pruned Mass [GeV]', 'Events/1.0')

    plotter.SetDataExpression('')
    plotter.SetDefaultExpr('fatjettau21')
    plotter.MakeCanvas('smearedup_tau21', 25, 0, 1.5, '#tau_{2}/#tau_{1}', 'Events/1.0')

    plotter.SetMakeRatio(False)
    MakePlots(categories, cutList, [['fatjetDRGenW', 25, 0, 5, '#Delta R from Gen W', 'Events/1.0']])
    plotter.SetMakeRatio(True)

if __name__ == '__main__':
    RunPlots(['semilep'])
    if len(sys.argv) > 1 and sys.argv[1] == 'full':
        outbase = plotter.GetOutDirectory().Data().rstrip('/')
        for ending in ['_background',  '_morebackground',  '_midbackground',  '_up',  '_down',  '_topoff',  '_topsquared']:
            if not os.path.exists(outbase + ending):
                os.makedirs(outbase + ending)

        plotter.ResetConfig()
        plotter.AddDataFile('wscale_Data.root')
        plotter.ReadMCConfig('MCBackground.txt')
        plotter.SetOutDirectory(outbase + '_background')
        RunPlots(['semilep'])

        cuts.defaultMCWeight = 'mcFactors/topPtReweighting'
        plotter.SetOutDirectory(outbase + '_topoff')
        RunPlots(['semilep'])
        cuts.defaultMCWeight = 'mcFactors * topPtReweighting'
        plotter.SetOutDirectory(outbase + '_topsquared')
        RunPlots(['semilep'])
        cuts.defaultMCWeight = 'mcFactors * topPtReweighting'

        plotter.ResetConfig()
        plotter.AddDataFile('wscale_Data.root')
        plotter.ReadMCConfig('MCBackground_more.txt')
        plotter.SetOutDirectory(outbase + '_morebackground')
        RunPlots(['semilep'])

        plotter.ResetConfig()
        plotter.AddDataFile('wscale_Data.root')
        plotter.ReadMCConfig('MCBackground_mid.txt')
        plotter.SetOutDirectory(outbase + '_midbackground')
        RunPlots(['semilep'])

        plotter.ResetConfig()
        plotter.AddDataFile('wscale_Data.root')
        plotter.ReadMCConfig('MCBackground_up.txt')
        plotter.SetOutDirectory(outbase + '_up')
        RunPlots(['semilep'])

        plotter.ResetConfig()
        plotter.AddDataFile('wscale_Data.root')
        plotter.ReadMCConfig('MCBackground_down.txt')
        plotter.SetOutDirectory(outbase + '_down')
        RunPlots(['semilep'])


N = int(input())

matrix = [[] for _ in range(N)]

for i in range(N):

    matrix[i] = list(input())

box_horizon = []

box_vertical = []

for m in range(N):

    count = 1

    for n in range(1,N):

        if matrix[m][n-1] == '.' and matrix[m][n-1] == matrix[m][n]:

            count +=1

        else:

            if count >= 2:

                box_horizon.append(count)

            else:

                count = 1

        if n == N-1:

            if count >=2:

                box_horizon.append(count)

for b in range(N):

    count = 1

    for a in range(1,N):

        if matrix[a][b] == '.' and matrix[a-1][b] == matrix[a][b]:

            count +=1

        else:

            if count >=2:

                box_vertical.append(count)

            else:

                count = 1

        if a == N-1:

            if count >=2:

                box_vertical.append(count)

print(len(box_horizon),len(box_vertical))


import numpy as np
from numpy.linalg import inv, eigh

class KernelFDA:
    def __init__(self, n_components, kernel):
        self.kernel = kernel
        self.p = n_components

    def fit(self, X, y):
        self.X_fit = X

        n = len(y)
        n1 = np.sum(y)
        n0 = n - n1

        # Compute kernel matrix
        hXX = self.kernel(X, X)

        # Compute mean vectors
        M1 = 1/n1 * (hXX @ y)
        M0 = 1/n0 * (hXX @ (np.ones(n) - y))

        # Compute between-class scatter matrix
        M = np.outer((M1 - M0), (M1 - M0))

        # Compute within-class scatter matrix
        K1 = hXX[:, y == 1]
        K0 = hXX[:, y == 0]
        H1 = np.eye(n1) - 1/n1 * np.ones((n1, n1))
        H0 = np.eye(n0) - 1/n0 * np.ones((n0, n0))
        N = K1.dot(H1).dot(K1.T) + K0.dot(H0).dot(K0.T)

        epsilon = 0.0001
        # Add a small epsilon to prevent singularity of matrix N
        sigm = inv(N + epsilon * np.eye(X.shape[0])).dot(M)

        # Compute eigenvectors and eigenvalues
        eig_val, eig_vec = eigh(sigm)
        idx = eig_val.argsort()[::-1]  # sort eigenvalues in descending order (largest eigenvalue first)
        eig_val = eig_val[idx]
        eig_vec = eig_vec[:, idx]

        # Extract the first p eigenvectors
        self.A = eig_vec[:, :self.p]

    def transform(self, X):
        # Compute transformed data using kernel matrix and eigenvectors
        K = self.kernel(X, self.X_fit)
        return K @ self.A

    def fit_transform(self, X, y):
        self.fit(X, y)
        return self.transform(X)

    def predict(self, X):
        # Make predictions using transformed data
        # Note: This implementation assumes n_components = 1 for simplicity
        y_pred = np.where(self.transform(X) > 0, 1, 0)
        return y_pred


import configparser
from datetime import datetime
import os
from pyspark.sql.types import StructType as R, StructField as Fld, DoubleType as Dbl, StringType as Str, IntegerType as Int, DateType as Date, LongType as Lng, TimestampType as TST
from pyspark.sql import SparkSession
from pyspark.sql.functions import udf, col,  to_timestamp, monotonically_increasing_id
from pyspark.sql.functions import year, month, dayofmonth, hour, weekofyear, date_format

config = configparser.ConfigParser()
config.read('dl.cfg')

os.environ['AWS_ACCESS_KEY_ID']=config.get('AWS_ID', 'AWS_ACCESS_KEY_ID')
os.environ['AWS_SECRET_ACCESS_KEY']=config.get('AWS_SEC', 'AWS_SECRET_ACCESS_KEY')

def create_spark_session():
    """Creates a spark session
       with spark.jars and apache.hadoop packages

    Parameters:No arguments

    Returns: spark session named "spark"

    """
    spark = SparkSession \
        .builder \
        .config("spark.jars.packages", "org.apache.hadoop:hadoop-aws:2.7.0") \
        .getOrCreate()
    return spark

def process_song_data(spark, input_data, output_data):
    """import Song dataset extract columns and create songs and artist tables
    write those tables to parquet files

    Parameters:
    spark: name of spark session
    input_data: location of the source data s3 bucket
    output_data: location of the destination data s3 bucket

    Returns:
    writes songs table in parquet to output_data location + songs
    writes artist_table in parquet to output_dat location + artists

    """

    # Setting up the JSON table structure for the Song dataset
    song_dataset_schema= R([
        Fld("artist_id", Str()),
        Fld("artist_latitude", Dbl()),
        Fld("artist_longitude", Dbl()),
        Fld("artist_location", Str()),
        Fld("artist_name", Str()),
        Fld("song_id", Str()),
        Fld("title", Str()),
        Fld("duration", Dbl()),
        Fld("year", Str()),
        ])

    """get filepath to song data file
    use "song_data/*/*/*/*.json" for full dataset
    use "song_data/A/B/C/TRABCEI128F424C983.json" to pull a single record

    """
    song_data = input_data + "song_data/*/*/*/*.json"

    # read song data file with dataset_schema
    df = spark.read.json(song_data, schema=song_dataset_schema)

    # extract columns to create songs table
    songs_table = df.select('song_id', 'artist_id', 'year', 'duration')

    # drop duplicate rows in songs table
    songs_table = songs_table.dropDuplicates()

    # write songs table to parquet files partitioned by year and artist
    songs_table.write.mode('append').partitionBy('year', 'artist_id').parquet(output_data + "songs")

    # extract columns to create artists table
    artists_table = df.select('artist_id', 'artist_name', 'artist_location', 'artist_latitude', 'artist_longitude')

    # drop duplicate rows in artists table
    artists_table = artists_table.dropDuplicates()

    # write artists table to parquet files
    artists_table.write.mode('append').parquet(output_data + "artists")

def process_log_data(spark, input_data, output_data):
    """import log dataset extract columns create users, time, and songplays table

    Parameters:
    spark: name of spark session
    input_data: location of the source data s3 bucket
    output_data: location of the destination data s3 bucket

    Returns:
    writes users table in parquet to output_data location + users
    writes time_table in parquet to output_data location + time
    writes songplays table in parquest to output_data location + songplays

    """

    """Setting up the JSON table structure for the log dataset"""
    log_dataset_schema= R([
       Fld("artist", Str()),
       Fld("auth", Str()),
       Fld("firstName", Str()),
       Fld("gender", Str()),
       Fld("iteminSession", Lng()),
       Fld("lastName", Str()),
       Fld("length", Dbl()),
       Fld("level", Str()),
       Fld("location", Str()),
       Fld("method", Str()),
       Fld("page", Str()),
       Fld("registration", Dbl()),
       Fld("sessionId", Lng()),
       Fld("song", Str()),
       Fld("status", Lng()),
       Fld("ts", Lng()),
       Fld("userAgent", Str()),
       Fld("userId", Str()),
   ])

    # get filepath to log data file
    log_data = input_data + "log_data/*/*/*.json"

    # read log data file
    df = spark.read.json(log_data, schema = log_dataset_schema)

    # filter by actions for song plays
    df = df.filter(col("page") == 'NextSong')

    # extract columns for users table
    users_table = df.select(col("userId").alias("user_id"), col("firstName").alias("first_name"),
                            col("lastName").alias("last_name"), "gender", "level")

    # drop duplicate rows in users table
    users_table = users_table.dropDuplicates()

    # write users table to parquet files
    users_table.write.mode('append').parquet(output_data+"users")

    # create timestamp column from original timestamp column
    dfTimestamp = df.withColumn("start_time", to_timestamp(col("ts") /1000 ))
    dfTimestamp = dfTimestamp \
    .withColumn("hour", hour("start_time")) \
    .withColumn("day", dayofmonth("start_time")) \
    .withColumn("week", weekofyear("start_time")) \
    .withColumn("month", month("start_time")) \
    .withColumn("year", year("start_time")) \
    .withColumn("weekday", date_format("start_time", 'E'))

    # extract columns to create time table
    time_table = dfTimestamp.select(col("start_time"), \
                                    col("hour"), \
                                    col("day"), \
                                    col("week"), \
                                    col("month"), \
                                    col("year"), \
                                    col("weekday"))

    # drop duplicate rows in time table
    time_table = time_table.dropDuplicates()

    # write time table to parquet files partitioned by year and month
    time_table.write.mode('append').partitionBy("year","month").parquet(output_data+"time")

    # read in song data to use for songplays table
    song_data = input_data + "song_data/*/*/*/*.json"
    song_df = spark.read.json(song_data)

    # extract columns from joined song and log datasets to create songplays table
    songplays_table = song_df.join(df, song_df.artist_name==df.artist). \
        withColumn("songplay_id", monotonically_increasing_id()). \
        withColumn('start_time', to_timestamp(col("ts") /1000 )). \
        select("songplay_id",
        "start_time",
        col("userId").alias("user_id"),
        "level",
        "song_id",
        "artist_id",
        col("sessionId").alias("session_id"),
        col("artist_location").alias("location"),
        "userAgent",
        month(col("start_time")).alias("month"),
        year(col("start_time")).alias("year"))

    # drop duplicate rows in songplays table
    songplays_table = songplays_table.dropDuplicates()

    # write songplays table to parquet files partitioned by year and month
    songplays_table.write.mode('append').partitionBy('year', 'month').parquet(output_data + "songplays")

def main():
    """main function executes create_spark_session, process_song_data function, and
    process_log_data function with input_data and output_data defined.

    Parameters: No arguments

    Returns: No Returns

    """
    spark = create_spark_session()
    # input_data defines the location of the source s3 bucket
    input_data = "s3a://udacity-dend/"
    # output_data defines the location of the destination s3 bucket

    output_data = "s3a://udacity-dend"

    process_song_data(spark, input_data, output_data)
    process_log_data(spark, input_data, output_data)

if __name__ == "__main__":
    main()


import pickle as pkl
import os
import cv2
import networkx as nx
import matplotlib.pyplot as plt
import matplotlib.image as mpimg

with open('./vrd/test.pkl', 'rb') as infile:
	data_test = pkl.load(infile)

with open('./vrd/train.pkl', 'rb') as infile:
	data_train = pkl.load(infile)

with open('objects-dict-2.pkl', 'rb') as infile:
	objects = pkl.load(infile)

l = []
d = dict()
d_classes = dict()
counter = 0
for img in os.listdir("./test_model_images"):
	print (img)
	test_image = cv2.imread('./test_model_images/' + str(img))

	for i in data_test:
		if i!=None:
			if i['img_path'] == '../data/sg_dataset/sg_test_images/' + str(img):
				data = i
				break

	for i in data_train:
		if i!=None:
			if i['img_path'] == '../data/sg_dataset/sg_train_images/' + str(img):
				data = i
				break

	#print (data)
	for val, name in zip(data['boxes'], data['classes']):

		print (val, objects[name])
		cv2.rectangle(test_image, (val[0], val[1]), (val[0]+val[2], val[1]+val[3]), (255, 0, 0), 2)
		x = val[0]
		h = val[1]
		cv2.putText(test_image, objects[name], (x, h), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 255, 255))

		l.append(counter)
		d[counter] = {'boxes': val, 'classes': objects[name]}
		d_classes[counter] = str(objects[name])
		counter = counter + 1

	cv2.imshow('im', test_image)
	cv2.imwrite('annotated_image.png', test_image)
	cv2.waitKey(0)
	cv2.destroyAllWindows()

	G = nx.complete_graph(len(l))
	G.add_nodes_from(l)
	nx.set_node_attributes(G, name='label', values=d)
	for d in (G.node.data()):
		print (d)

	print (len(G.edges()))

	print (d_classes)
	print ("Graph created and saved")
	counter = 0

	H = nx.relabel_nodes(G, d_classes, copy = True)
	nx.draw(H, with_labels = True)
	plt.savefig('scene_graph_beginning.png')
	plt.show()


# We are given an array containing positive and negative numbers.
# Suppose the array contains a number ‘M’ at a particular index.
# Now, if ‘M’ is positive we will move forward ‘M’ indices and
# if ‘M’ is negative move backwards ‘M’ indices. You should assume that
# the array is circular which means two things:

# 1. If, while moving forward, we reach the end of the array,
# we will jump to the first element to continue the movement.

# 2. If, while moving backward, we reach the beginning of the array,
# we will jump to the last element to continue the movement.

# Input: [2, 2, -1, 2]
# Output: true
# Explanation: The array has a cycle among indices: 1 -> 3 -> 1

# Input: [2, 1, -1, -2]
# Output: false
# Explanation: The array does not have any cycle.

arr = [2,1,-1,-2]
arr1 = [1, 2, -1, 2, 2]

arr2 =  [2, 2, -1, 2]
def circular_array_loop_exists(arr):
    for i in range(len(arr)):
        slow ,fast= i,i
        while (slow!=fast):
            if fast<0:
                fast = fast - arr[fast]
            else:
                fast = fast - arr[fast]
        if slow==fast:
            return "array has cycle"
    else:
        return "array doesn't have cycle"

print(circular_array_loop_exists(arr))

print(circular_array_loop_exists(arr1))

print(circular_array_loop_exists(arr2))


"""
Unit tests for FileBackedSparseStore.
"""
import json
import os
import random
import shutil
import string
import time
import tempfile
from typing import List, IO
from unittest import TestCase
from contextlib import contextmanager
import gzip
import pytest

from nbdb.common.data_point import DataPoint
from nbdb.store.file_backed_sparse_store import FileBackedSparseStore

def _random_string() -> str:
    length = random.randint(4, 32)
    return ''.join(random.choices(
        string.ascii_letters + string.digits, k=length))

def _generate_data_points(datasource: str, num: int) -> List[DataPoint]:
    results = []
    field = _random_string()
    start_time = time.time_ns()
    for i in range(num):
        data_point = DataPoint(
            datasource=datasource,
            field=field,
            tags={},
            epoch=start_time + i*10**3,
            server_rx_epoch=start_time + i**10**3,
            value=random.randint(0, 2**32))
        results.append(data_point)
    return results

class UncompressedDeps:
    """
    Test FileBackedSparseStore without compression.
    """

    @staticmethod
    def create_store(directory: str) -> FileBackedSparseStore:
        """Create an uncompressed filestore."""
        return FileBackedSparseStore(directory, compress=False)

    @staticmethod
    @contextmanager
    def open_sparse_file(path) -> IO:
        """Open an uncompressed sparse file."""
        file = open(path, 'r')
        try:
            yield file
        finally:
            file.close()

class CompressedDeps:
    """
    Test FileBackedSparseStore with compression.
    """

    @staticmethod
    def create_store(directory: str) -> FileBackedSparseStore:
        """Create a compressed filestore."""
        return FileBackedSparseStore(directory, compress=True)

    @staticmethod
    @contextmanager
    def open_sparse_file(path) -> IO:
        """Open a compressed
        sparse file."""
        file = gzip.open(path, 'rt')
        try:
            yield file
        finally:
            file.close()

# pylint: disable=redefined-outer-name
@pytest.fixture()
def test_dir():
    """Temporary directory resource for tests."""
    test_dir = os.path.join(tempfile.mkdtemp(), "filestore")
    yield test_dir
    shutil.rmtree(test_dir)

@pytest.fixture(params=[CompressedDeps(), UncompressedDeps()])
def deps(request):
    """Dependencies for tests."""
    return request.param

class TestFileBackedSparseStore:
    """
    Base test class for FileBackedSparseStore.
    """

    @staticmethod
    def _verify_sparse_store_json(deps, test_dir, filename: str, num_dps: int) -> None:
        filepath = os.path.join(test_dir, filename)
        with deps.open_sparse_file(filepath) as file:
            lines = file.readlines()
            parsed = [json.loads(line) for line in lines]
            assert len(parsed) == num_dps

    @staticmethod
    def test_empty_store(deps, test_dir):
        """
        Verify operations on empty store.
        """
        sparse_store = deps.create_store(test_dir)
        sparse_store.flush()
        sparse_store.close()

        # Manifest file must exist
        assert os.path.isfile(os.path.join(test_dir, "MANIFEST"))
        assert len(
            FileBackedSparseStore.parse_datasources_from_manifest(
                test_dir)
            ) == 0

        assert sparse_store.points_written == 0

        # Close is idempotent
        sparse_store.close()
        # Flush will fail for closed stores
        with pytest.raises(ValueError):
            sparse_store.flush()
        # Write will fail for closed stores
        data_point, *_ = _generate_data_points("ds", 1)
        with pytest.raises(ValueError):
            sparse_store.write(data_point)
        # Creating another store in the same directory will fail
        with pytest.raises(ValueError):
            FileBackedSparseStore(test_dir, compress=False)

    @staticmethod
    def test_writes(deps, test_dir):
        """
        Verify write operations on file backed sparse store.
        """
        sparse_store = deps.create_store(test_dir)
        ds1_dps = _generate_data_points("ds1", 101)
        ds2_dps = _generate_data_points("ds2", 99)

        for data_point in ds1_dps:
            sparse_store.write(data_point)
        sparse_store.flush()
        ds_map = \
            FileBackedSparseStore.parse_datasources_from_manifest(test_dir)
        assert len(ds_map) == 1
        ds1_info = ds_map["ds1"]
        assert ds1_info.points_written == 101

        # Write ds2_dps
        for data_point in ds2_dps:
            sparse_store.write(data_point)
        sparse_store.flush()
        ds_map = \
            FileBackedSparseStore.parse_datasources_from_manifest(test_dir)
        assert len(ds_map) == 2
        ds2_info = ds_map["ds2"]
        assert ds2_info.points_written == 99
        assert sparse_store.points_written == 200
        # Directory should have a total of 3 files
        assert len(os.listdir(test_dir)) == 3

        sparse_store.close()
        # All files must be closed
        for ds_file in sparse_store.datasource_to_file.values():
            assert ds_file.file.closed
        # Close is idempotent
        sparse_store.close()

        TestFileBackedSparseStore._verify_sparse_store_json(
            deps, test_dir, ds1_info.filename, 101)
        TestFileBackedSparseStore._verify_sparse_store_json(
            deps, test_dir, ds2_info.filename, 99)

        # Manifest can be read after close
        ds_map = \
            FileBackedSparseStore.parse_datasources_from_manifest(test_dir)
        assert len(ds_map) == 2
        # Flush will fail for closed stores
        with pytest.raises(ValueError):
            sparse_store.flush()
        # Write will fail for closed stores
        with pytest.raises(ValueError):
            sparse_store.write(ds1_dps[0])


import string

def build_word_set( input_file ):

    word_set = set()

    for line in input_file:

        # append words to list
        word_lst = line.strip().split()

        # remove punctuation
        word_lst = [w.lower().strip(string.punctuation) for w in word_lst]

        for word in word_lst:

            if word != "":

                # add the list if it has space
                word_set.add( word )

    return word_set

def compare_files( file1, file2 ):

    # Build two sets:
    word_set1=build_word_set(file1)
    word_set2=build_word_set(file2)
    # Display the total number of unique words between the
    # two files.  If a word appears in both files, it should
    # only be counted once.
    unique_word_count=word_set1|word_set2
    print("Total unique words:", len(unique_word_count))

    # Display the number of unique words which appear in both
    # files.  A word should only be counted if it is present in
    # both files.
    unique_word_in_both_count=word_set1&word_set2
    print("Unique words that appear in both files:", len(unique_word_in_both_count))

######################################################################

f1 = open( "document1.txt" )
f2 = open( "document2.txt" )

compare_files( f1, f2 )

f1.close()
f2.close()


import streamlit as st
import pandas as pd
import folium
from matplotlib import image
from geopy.geocoders import Nominatim
from sklearn.neighbors import DistanceMetric
from streamlit_folium import folium_static
from PIL import Image

# Create a multiselect widget for the user to choose a page
page_options = ['Home Page', 'Pub Locations', 'Find the nearest Pub']
selected_page = st.sidebar.selectbox('Select a page', page_options)

pubs_df = pd.read_csv('pub_df.csv')

# Use if else statements to display the selected page

###################PAGE-1#####################

if selected_page == 'Home Page':
    st.title(':beers: :red[Welcome to the Pub Finder Web Application]:beers:')
    st.write('This application displays information and statistics about pubs in the UK.')
    # Add an image
    image = Image.open('pub2.jpg')
    st.image(image)
    st.write(f'Total number of pubs: {len(pubs_df)}')
    st.write(f'Number of cities: {pubs_df["city"].nunique()}')
    st.write(f'Number of postal codes: {pubs_df["postcode"].nunique()}')
    city_count = pubs_df['city'].value_counts()
    highest_city = city_count.nlargest(1)
    st.write(f'City with highest number of pubs: {highest_city.index[0]} ({highest_city.values[0]} pubs)')

    st.subheader("About me :point_down:")
    st.write("Nandini Shukla :wave:")
    st.write("Data Science intern @Innomatics research lab :computer:")

    st.subheader("Contect me :coffee:")
    st.markdown("Gmail  - nandini0212shukla@gmail.com")
    st.markdown("Github  - https://github.com/OkayMini")
    st.markdown("Linkedin  - https://www.linkedin.com/in/nandinishuklads/")

###################PAGE-2#####################

elif selected_page == 'Pub Locations':
    # Set up the app layout
    st.title(":beer: :blue[Pub Locations] :beer:")

    # Get user inputs for postal code and local authority
    location_pc = st.text_input('Enter your Postal Code:', 'SW1A 2AA')
    location_la = st.text_input('Enter your Local Authority:', 'Westminster')

    # Geocode the user's location
    geolocator = Nominatim(user_agent='my_app')
    location_data = geolocator.geocode(f'{location_pc} {location_la}', exactly_one=False)[0]
    lat, lon = location_data.latitude, location_data.longitude

    # Create a folium map object centered on the user's location
    m = folium.Map(location=[lat, lon], zoom_start=13)

    # Filter the pubs based on the chosen area
    filtered_pubs = pubs_df[(pubs_df['postcode'] == location_data.raw.get('postcode', '')) |
                            (pubs_df['city'] == location_data.raw.get('city', '')) |
                            (pubs_df['local_authority'] == location_la)]

    # Add a marker for each pub in the filtered dataset
    for i, pub in filtered_pubs.iterrows():
        folium.Marker(location=[pub['latitude'], pub['longitude']],
                    popup=pub['name']).add_to(m)

    # Display the map using Streamlit
    st.markdown('### Pub Locations')
    st.write(f'Number of pubs in {location_pc} {location_la}: {len(filtered_pubs)}')
    folium_static(m)

    # Display the list of pubs in the filtered dataset
    st.markdown('### List of Pubs')
    for i, pub in filtered_pubs.iterrows():
        st.write(pub['name'])

###################PAGE-3#####################

elif selected_page == 'Find the nearest Pub':
    # Ask the user to enter their latitude and longitude
    st.title('Find the Nearest Pubs')
    lat = st.number_input('Enter your latitude:')
    lon = st.number_input('Enter your longitude:')

    # Find the nearest 5 pubs using Haversine distance
    dist = DistanceMetric.get_metric('haversine')
    distances = dist.pairwise(pubs_df[['latitude', 'longitude']], [[lat, lon]])
    pubs_df['Distance'] = distances[:, 0]
    nearest_pubs = pubs_df.nsmallest(5, 'Distance')

        # Display a map with the nearest pubs
    st.title('Nearest Pubs')
    m = folium.Map(location=[lat, lon], zoom_start=13)
    for i, pub in nearest_pubs.iterrows():
        icon = folium.Icon(icon='beer', prefix='fa')
        folium.Marker(location=[pub['latitude'], pub['longitude']],
                    popup=pub['name'], icon=icon).add_to(m)
    folium_static(m)

    # Display a table with the nearest pubs
    st.title('Nearest Pubs List')
    st.write(nearest_pubs[['name', 'address', 'Distance']])


import logging

def batcher(generator, batch_func, batch_size):
    buf = []
    for item in generator:
        buf.append(item)
        if len(buf) >= batch_size:
            output = batch_func(buf)
            for item in buf:
                yield (item, output)
            buf = []
    if buf:
        output = None
        try:
            output = batch_func(buf)
        except:
            logging.exception("batch function failed!")

        if output:
            for item in buf:
                yield (item, output)


import sys,torch,torchvision,logging
import matplotlib.pyplot as plt
import numpy as np
from pathlib import Path
from torch.utils.data import Dataset, DataLoader
import torch.nn as nn
#logging.basicConfig(level=logging.DEBUG)
logging.info("torch: {0} ,  torchvision:{1}".format(torch.__version__,torchvision.__version__))
class Learner(object):
    def __init__(self,train_dataset,test_dataset,model):
        logging.debug("----"+sys._getframe().f_code.co_name+"----")
        #torch的Dataset
        self.__train_dataset = None
        self.__test_dataset = None
        #优化器
        self.__optimizer =None
        self.__optim_param={}
        #损失函数
        self.__criterion=None
        self.__crit_param={}
        #训练相关
        self.__train_param={}
        self.__train_dataloader=None
        #测试相关
        self.__test_param={}
        self.__test_dataloader=None
        # 模型
        self.__model=None
        #
        self.__device=torch.device("cpu")
        self.__trained=False
        if (isinstance(train_dataset,Dataset)):
            self.__train_dataset=train_dataset
        else:
            raise(TypeError("dataset必须为pythrch Dataset类型"))
        if (isinstance(test_dataset,Dataset)):
            self.__test_dataset=test_dataset
        else:
            raise(TypeError("dataset必须为pythrch Dataset类型"))
        if (isinstance(model,nn.Module)):
            self.__model=model
        else:
            raise(TypeError("model必须为pytorch Module类型"))
    def setCriterion(self,crit,**param):
        logging.debug("----"+sys._getframe().f_code.co_name+"----")
        #设置损失函数
        #https://pytorch.org/docs/stable/_modules/torch/nn/modules/loss.html
        if issubclass(crit,nn.modules.loss._Loss):
            self.__criterion=crit
            self.__crit_param=param
        else:
            raise(TypeError("损失函数必须是 _Loss 子类"))
            #crit(**param)
    def setOptimizer(self,opt,**param):
        logging.debug("----"+sys._getframe().f_code.co_name+"----")
        #设置优化器
        if issubclass(opt,torch.optim.Optimizer):
            self.__optimizer=opt
            self.__optim_param=param
        else:
            raise(TypeError("优化器必须是 Optimizer 子类"))

    def setTrainParam(self,key,value):
        logging.debug("----"+sys._getframe().f_code.co_name+"----")
         # 设置训练的参数
        self.__train_param[key]=value
    def setTestParam(self,key,value):
        logging.debug("----"+sys._getframe().f_code.co_name+"----")
         # 设置训练的参数
        self.__test_param[key]=value

    def checkDevice(self,device="auto"):
        logging.debug("----"+sys._getframe().f_code.co_name+"----")
        #检查cpu还是gpu
        if device=="auto":
            if torch.cuda.is_available():
                self.__device=torch.device("cuda")
        elif device=="cuda":
                self.__device=torch.device("cuda")
        logging.info("Use Device:{}".format(self.__device))

    def __checkTrainParam(self):
        logging.debug("----"+sys._getframe().f_code.co_name+"----")
        # 检查参数 lr,bs
        check=set(['lr','bs'])
        print(self.__train_param.keys())
        if not (check <= self.__train_param.keys()):
            raise(ValueError("必须要设置lr和bs"))
    def showSummary(self):
        logging.debug("----"+sys._getframe().f_code.co_name+"----")
        #打印汇总
        print("Device:{}".format(self.__device))
        print("Optimizer:{},Param:{}".format(self.__optimizer,self.__optim_param))
        print("Criterion:{},Param:{}".format(self.__criterion,self.__crit_param))
        print("TrainParam:{}".format(self.__train_param))

    def train(self,fetch,epoches=10,device="auto",show_summary=False):

        logging.debug("----"+sys._getframe().f_code.co_name+"----")
        self.__losses=[]
        self.__trained=True
        #检查参数
        self.__checkTrainParam()
        #读取数据,默认不打乱
        shuffle= self.__train_param["shuffle"] if "shuffle" in self.__train_param.keys() else False
        self.__train_dataloader=DataLoader(dataset = self.__train_dataset,
                                            batch_size = self.__train_param['bs'],
                                            shuffle = shuffle)  # 将数据打乱
        #判断gpu
        self.checkDevice(device=device)

        # 显示汇总信息
        if (show_summary):
            self.showSummary()

        # 模型
        self.__model=self.__model.to(self.__device)
        self.__model.train()
        # 优化器
        train_opti=self.__optimizer(self.__model.parameters(),**self.__optim_param)
        # 损失函数
        train_crit=self.__criterion(**self.__crit_param).to(self.__device)
        for epoch in range(epoches):
            #循环
            for i,data in enumerate(self.__train_dataloader):
                logging.info("epoch:{},batch:{}".format(epoch+1,i))
                #批次
                x,y=fetch(data)
                train_opti.zero_grad()
                y_hat= self.__model(x.to(self.__device))
                loss = train_crit(y_hat, y.to(self.__device))
                loss.backward()
                train_opti.step()
                self.__losses.append(loss.cpu().data.item())
            print("epoch:{},avg loss:{}".format(epoch+1,np.mean(self.__losses)))
        print("训练完成")
    def test(self,fetch,metrics):
        logging.debug("----"+sys._getframe().f_code.co_name+"----")

        if  self.__trained==False :
            print("请先运行train")
            return
        bs= self.__test_param["bs"] if "bs" in self.__test_param.keys() else self.__train_param['bs']
        self.__test_dataloader=DataLoader(dataset = self.__test_dataset,
                                            batch_size = bs,
                                            shuffle = False)
        self.__model=self.__model.to(self.__device)
        self.__model.eval()
        correct =total = 0
        for data in self.__test_dataloader:
            inputs,labels=fetch(data)
            outputs = self.__model(inputs.to(self.__device)).cpu()
            predicted = metrics(outputs)
            total+=labels.size(0)
            correct+=(predicted == labels).sum()
        print('准确率: %6.f %%' % (100 * correct / total))
    def saveModel(self,save_path):
        torch.save(self.__model.state_dict(), save_path)
    def loadModel(self,load_path):
        self.__model.load_state_dict(torch.load(load_path))
    def showLossPlt(self):
        plt.xlabel('batch #')
        plt.ylabel('Loss')
        plt.plot(self.__losses)
        plt.show()


from django.contrib import admin
from django.urls import path,include
from django.conf import settings
from django.conf.urls.static import static
from . import views,CreditOfficer_Views,Office_Views,Supervisor_Views,Admin_Views,Customer_Views

urlpatterns = [
    path('admin/', admin.site.urls),
    path('base', views.Base, name='base' ),
    path('pdf/', include('app.urls')),

    # login paths
    path('', views.Login, name='login'),
    path('login', views.DoLogin, name='dologin'),
    path('logout', views.DoLogout, name='logout'),

    # Admin paths
    path('Admin/Home', Admin_Views.AdminHome, name='adminhome'),
    path('Admin/Supervisor/Add', Admin_Views.AddSupervisor, name='addsupervisors'),
    path('Admin/Credit/Add', Admin_Views.AddCreditOfficer, name='addcreditofficers'),
    path('Admin/Customers/Add', Admin_Views.AddCustomer, name='addcustomers'),
    path('Admin/Customers/View', Admin_Views.ViewCustomer, name='viewcustomers'),

    path('Admin/Office/Add', Admin_Views.AddOffice, name='addoffice'),
    path('Admin/Customer/Loan', Admin_Views.CustomerLoan, name='customerloan'),
    path('Admin/Customer/Approve/Loan/<str:pk>', Admin_Views.CustomerApproveLoan, name='customerapproveloan'),
    path('Admin/Customer/Disapprove/Loan/<str:pk>', Admin_Views.CustomerDisapproveLoan, name='customerdisapproveloan'),
    path('create_pdf', Admin_Views.CustomerPdfReport, name="create_pdf"),

    # CreditOfficer paths
    path('Credit/Home', CreditOfficer_Views.CreditOfficerHome, name='credithome'),
    path('interest_calculator/', CreditOfficer_Views.interest_calc, name = 'creditcustomersloan'),
    path('interest_calculator/results/', CreditOfficer_Views.results_interest, name = 'creditviewcustomers'),

    # Customer paths
    path('Customer/Home', Customer_Views.CustomerHome, name='customerhome'),
    path('Customer/Apply/Loan', Customer_Views.CustomerApplyLoan, name='customerapplyloan'),
    path('Customer/Apply/Loan/save', Customer_Views.CustomerApplyLoanSave, name='customerapplyloansave'),

    # Office paths
    path('Office/Home', Office_Views.OfficeHome, name='officehome'),

    # Supervisor paths
    path('Supervisor/Home', Supervisor_Views.SupervisorHome, name='supervisorhome'),
]

urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)

# urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)


### This script will be run from the command line to get cross-session registration
import sys
import os
from pathlib import Path
import click
import json
import subprocess

from helper_fx import Preprocess, setup_logger

@click.command()
@click.option("--config-file", type=str, default="config.json", help="A file containing config options. If not present are parsed correctly the script will exit.")
@click.option("--get-metafile", "-m", type=bool, is_flag=True, help="If selected attempts to download metafile from Azure")
@click.option("--animals", "-a", type=str, default="", help="List of animals to be processed")
@click.option("--dates", "-d", type=str, default="", help="List of dates to be processed")
@click.option("--use-fast-dir", "-f", type=bool, is_flag=True, help="Path to fast directory, important when using with a VM and file share to speed up")
@click.option("--overwrite", type=bool, is_flag=True, help="Choose if you want the option to overwrite files")
@click.option("--delete_intermediates", "-X", type=bool, is_flag=True, help="When selected, will delete raw data and imageJ files")
def run_processing(config_file, get_metafile, animals, dates, use_fast_dir, overwrite, delete_intermediates):
    print("run")

        # finds and opens config file
    print(f"The config file is {config_file}")
    f = open(config_file)
    config_data = json.load(f)

    # Initializes class
    preprocess = Preprocess(config_data, use_fast_dir, overwrite, delete_intermediates)

    # Sets project_dir (and create if does not exist)
    preprocess.set_project_dir()

    # Sets up logger and make directory if needed
    preprocess.set_logger()

    # Gets metafile if requested
    if get_metafile:
        preprocess.get_metafile()

        # Reads metafile in
    preprocess.read_metafile()

    # Parses animals and check to make sure only one is given
    preprocess.parse_animals(animals)
    if len(preprocess.animals) > 1:
        preprocess.logger.info("Exiting as more than one animal given. This script can only process one at a time")
        sys.exit(2)

    # Parses dates and checks to make sure >1 given
    preprocess.parse_dates(dates)
    if len(preprocess.dates) < 2:
        preprocess.logger.info("Exiting as only one date given. Script is designed to look at 2 or more sessions")
        sys.exit(2)

    # Sets up paths and directories
    preprocess.define_root()
    preprocess.define_nwb_paths()
    preprocess.define_multisession_paths() # need to define multiple paths for rawdata and imagej data using dates

# need to decide on file structure
# function to create
# then look for existing registered tiffs

if __name__ == "__main__":
    print("processing stuff")
    run_processing()


"""add_explorer_field_to_dataset

Revision ID: 16_b65dad1f2a7e
Revises: 15_78103925d379
Create Date: 2021-06-02 14:52:17.931804

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = "16_b65dad1f2a7e"
down_revision = "15_78103925d379"
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column("dataset", sa.Column("explorer_url", sa.String(), nullable=True))
    op.create_index(op.f("ix_dataset_explorer_url"), "dataset", ["explorer_url"], unique=False)
    op.execute(
        "UPDATE dataset SET explorer_url=deployment_directory.url FROM deployment_directory WHERE "
        "deployment_directory.dataset_id=dataset.id;"
    )  # noqa E501
    op.drop_constraint("deployment_directory_dataset_id_fkey", "deployment_directory")
    op.drop_table("deployment_directory")
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table(
        "deployment_directory",
        sa.Column("id", sa.VARCHAR(), autoincrement=False, nullable=False),
        sa.Column("dataset_id", sa.VARCHAR(), autoincrement=False, nullable=False),
        sa.Column("url", sa.VARCHAR(), autoincrement=False, nullable=True),
        sa.Column(
            "created_at", postgresql.TIMESTAMP(), server_default=sa.text("now()"), autoincrement=False, nullable=False
        ),
        sa.Column(
            "updated_at", postgresql.TIMESTAMP(), server_default=sa.text("now()"), autoincrement=False, nullable=False
        ),
        sa.ForeignKeyConstraint(["dataset_id"], ["dataset.id"], name="deployment_directory_dataset_id_fkey"),
        sa.PrimaryKeyConstraint("id", name="deployment_directory_pkey"),
    )
    op.execute('CREATE EXTENSION IF NOT EXISTS "uuid-ossp";')
    op.execute(
        "INSERT into deployment_directory (id, dataset_id, url, created_at, updated_at) SELECT uuid_generate_v4(), id, "
        "explorer_url, NOW(), NOW() FROM dataset WHERE dataset.explorer_url is not null;"
    )  # noqa E501
    op.drop_index(op.f("ix_dataset_explorer_url"), table_name="dataset")
    op.drop_column("dataset", "explorer_url")
    # ### end Alembic commands ###


#!/usr/bin/env python
# coding: utf-8

import torch
from torchvision.datasets import ImageNet
from torch.utils.data import DataLoader, Dataset
from torchvision import transforms
import torchvision

import numpy as np
from tqdm import tqdm
import matplotlib.pyplot as plt
import seaborn as sns

import os
from PIL import Image
import json

from circuit_explorer.utils import load_config
from circuit_explorer.utils import get_layers_from_model

from torchvision import transforms, datasets
from circuit_explorer.data_loading import rank_image_data, single_image_data
from circuit_explorer.receptive_fields import receptive_field, receptive_field_for_unit

from collections import defaultdict
from circuit_explorer.target import layer_saver
from heapq import heappush, heappushpop

from circuit_explorer.target import layer_saver
from circuit_explorer.data_loading import default_unnormalize

import hdbscan
from dataset.Dataset_Utils import ImageNetReceptiveField, ImageNet2

position = [6, 6]

def k_layer_activations_from_dataloader(layers,dataloader,model,num_units=[384],k=300,position=[6,6],batch_size=64, device="cuda:0"):

    device = next(model.parameters()).device
    topk_activations = {}
    layer_num_units = {}

    layer_activations = {}
    if isinstance(layers,str):
        layers = [layers]
    for i in layers:
        num_units = get_layers_from_model(model)[i].out_channels
        layer_num_units[i] = num_units
        for unit in range(num_units):
            topk_activations[f'{i}_{unit}'] = []

    for dl_idx, (data, labels) in enumerate(tqdm(dataloader)):
        ds_idxs = list(range(dl_idx*batch_size, (dl_idx+1)*batch_size))

        images = data.to(device)
        with layer_saver(model, layers) as extractor:
            batch_layer_activations = extractor(images)
            for i in layers:
                for unit in range(layer_num_units[i]):
                    unit_activations = list(batch_layer_activations[i][:, unit, position[0], position[1]].detach().to('cpu').numpy())
                    tuples_to_add = zip(unit_activations, ds_idxs)

                    for tup in tuples_to_add:
                        if len(topk_activations[f'{i}_{unit}']) < k:
                            heappush(topk_activations[f'{i}_{unit}'], tup)
                        else:
                            heappushpop(topk_activations[f'{i}_{unit}'], tup)

    return topk_activations

def cluster(target_layer_activations, layer, unit, model, imagenet_train_dataset, min_cluster_size=50, device="cuda:0"):
    original_activations = target_layer_activations[f'{layer}_{unit}']
    activations, indices = list(zip(*original_activations))

    sorted_idxs = np.flip(np.argsort(activations))
    sorted_activations = np.array(activations)[sorted_idxs]
    sorted_indices = np.array(indices)[sorted_idxs]

    values, idxs = sorted_activations, sorted_indices

    layer_activations = {}
    if isinstance(layer, str):
        layers = [layer]
    for i in layers:
        layer_activations[i] = []

    for i in range(len(idxs)):
        data = imagenet_train_dataset[idxs[i]]
        images = data[0].to(device)

        with layer_saver(model, layers) as extractor:
            batch_layer_activations = extractor(images.unsqueeze(0))
            for i in layers:
                layer_activations[i].append(batch_layer_activations[i].detach().to('cpu'))

    for i in layers:
        layer_activations[i] = torch.cat(layer_activations[i])

    cluster_points = layer_activations[layer][:,:,position[0],position[1]]

    clusterer = hdbscan.HDBSCAN(min_cluster_size=min_cluster_size)
    labels = clusterer.fit_predict(cluster_points.numpy())
    return labels, values, idxs

def get_unit_clustering_statistics(target_layer_activations, layer, model, imagenet_train_dataset, num_filters=256, min_cluster_size=50, device="cuda:0"):
    unit_dict = {}
    for u in tqdm(range(num_filters)):
        labels, _, _ = cluster(target_layer_activations, layer, u, model, imagenet_train_dataset, min_cluster_size=min_cluster_size, device=device)

#         num_0s = len(np.where(labels==0)[0])
#         num_1s = len(np.where(labels==1)[0])
#         num_neg_1s = len(np.where(labels==-1)[0])
        unit_dict[u] = {val:len(np.where(labels==val)[0]) for val in np.unique(labels)}
    return unit_dict

def cluster_and_plot_unit(target_layer_activations, layer, model, imagenet_train_dataset, unit, name, min_cluster_size=50, device="cuda:0"):
    labels, values, idx = cluster(target_layer_activations, layer, unit, model, imagenet_train_dataset, min_cluster_size=min_cluster_size, device=device)

    plt.figure(figsize=(12, 12))
    num_images = 16
    for i in range(num_images):
        plt.subplot(4, 4, i+1)
        if i < int(num_images/2):
            plt.imshow(default_unnormalize(imagenet_train_dataset[random.choice(idxs[np.where(labels == 0)])][0]).numpy().transpose((1, 2, 0)))
        else:
            plt.imshow(default_unnormalize(imagenet_train_dataset[random.choice(idxs[np.where(labels == 1)])][0]).numpy().transpose((1, 2, 0)))
        plt.axis('off')
        plt.tick_params(axis='both', left=False, top=False, right=False, bottom=False, labelleft=False, labeltop=False, labelright=False, labelbottom=False)
        plt.tight_layout()

    plt.savefig(name)

def main(FLAGS):

    device = FLAGS.device

    config_file = FLAGS.config_file
    config = load_config(config_file)

    model = config.model
    model = model.to(device)

    all_layers = get_layers_from_model(model)
    all_layers.keys()

    model.eval().cuda()
    mean = (0.485, 0.456, 0.406)
    std = (0.229, 0.224, 0.225)

    preprocessing = transforms.Compose(
                [
                    transforms.Resize(256),
                    transforms.CenterCrop(224),
                    transforms.ToTensor(),
                    transforms.Normalize(mean, std),
                ]
            )

    position = [6,6]
    # layer = 'features.8'
    # unit = 53

    #dog/metal feature
    layer = FLAGS.layer

    all_recep_field_params = receptive_field(model.features, (3,224,224))
    recep_field = receptive_field_for_unit(all_recep_field_params, layer, position)

    # imagenet2_train_dataset = ImageNet2('image_data/imagenet_2', preprocessing,
    #                                                 recep_field=recep_field)

    # imagenet2_train_dataloader = DataLoader(
    #             imagenet2_train_dataset,
    #             batch_size=128,
    #             num_workers=1,
    #             shuffle=False,
    #             drop_last=False,
    #             pin_memory=True
    #         )

    imagenet_train_dataset = ImageNetReceptiveField(FLAGS.data_path, "train", preprocessing,
                                                    recep_field=recep_field)

    imagenet_train_dataloader = DataLoader(
                imagenet_train_dataset,
                batch_size=FLAGS.batch_size,
                num_workers=8,
                shuffle=False,
                drop_last=False,
                pin_memory=True
            )

    target_layer_activations = k_layer_activations_from_dataloader(layer,imagenet_train_dataloader,
                                                                model, k=FLAGS.k, batch_size=FLAGS.batch_size,
                                                                device=device)

    num_filters = all_layers[layer].out_channels
    unit_dict = get_unit_clustering_statistics(target_layer_activations, layer, model,
                                            imagenet_train_dataset, num_filters=num_filters, min_cluster_size=FLAGS.min_cluster_size,
                                            device=device)

    layer_num = int(layer.split('.')[-1])
    torch.save(target_layer_activations, f'outputs/layer_{layer_num}/layer_{layer_num}_image_net_{FLAGS.k}.pt')
    torch.save(unit_dict, f'outputs/layer_{layer_name}/cluster_unit_stats_k={FLAGS.k}_mcs={FLAGS.min_cluster_size}.pt')

    if FLAGS.plot:
        for unit in units:
            cluster_and_plot_unit(target_layer_activations, layer, model, imagenet_train_dataset, unit,
                                    f'outputs/layer_{layer_name}/unit_{unit}_clusters.png', min_cluster_size=FLAGS.min_cluster_size,
                                    device=device)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
                prog='HDBScan_Clustering',
                description='clusters specified layer and gives unit features')
    parser.add_argument("--layer", default="features.10", type=str, help="specified layer to collect top k activations at")
    parser.add_argument("--k", default=1000, type=int, help="number of activations to collect and cluster with")
    parser.add_argument("--batch-size", default=64, type=int, help="batch size")
    parser.add_argument("--min-cluster-size", default=50, type=int, help="HDB clustering min cluster size")
    parser.add_argument("--device", default="cuda:0", type=str, help="device to run on")
    parser.add_argument("--plot", default=False, type=bool, help="whether to plot and save given units")
    parser.add_argument('--units', nargs='+', help='which units to plot the two classes of')
    parser.add_argument("--config-file", default="configs/alexnet_sparse_config.py", type=str,
        help="config file specifying model to run (default: alexnet-sparse)"
        )
    parser.add_argument("--data-path", default="image_data/imagenet", type=str,
        help="where the imagenet data is located")

    FLAGS = parser.parse_args()
    main(FLAGS)


""" check_ea1.py Assumes input is utf8-unicode, and similarly writes.
    03-24-2017.
    01-23-2018.  Handles combining codes
    06-30-2023.  Revise for python3
"""

import re,sys
import codecs, unicodedata

def make_combining_chars():
 chars = [
  r'\u0300', # combining grave accent
  r'\u0301', # combining acute accent
  r'\u0302', # combining circumflex accent
  r'\u0303', # combining tilde
  r'\u0304', # combining macron
 ]
 ans = []
 for x in chars:
  # ref: //stackoverflow.com/questions/2828284/conversion-of-strings-like-uxxxx-in-python
  # y = x.decode('unicode-escape')
  # AttributeError: 'str' object has no attribute 'decode'
  y = bytes(x,"ascii").decode('unicode-escape')
  ans.append(y)
 if False:
  for key in ans:
   ords = [r"\u%04x" % ord(c) for c in key]
   ordstr = '.'.join(ords)
   names = [unicodedata.name(c) for c in key]
   namestr = ' + '.join(names)
   out = "%s  (%s)  := %s" %(key,ordstr,namestr)
   print(out.encode('utf-8'))
  exit(0)
 return ans

combining_chars = make_combining_chars() # a list

def update_asdict(line,asdict):
 if line == '':
  return
 parts = []
 prev = None
 for ic,c in enumerate(line):
  if ic == 0:
   prev = c
  elif c in combining_chars:
   prev = prev + c
  else:
   # not a combining character and not the first character
   parts.append(prev)
   prev = c
  if ord(c) == 8206: # \u200e left-to-right mark:
   print('left-to-right-mark at character position',ic+1,'in line\n' ,line)
   print(len(line))
   atemp =[]
   for jc in range(ic-5,ic+5):
    xc = line[jc]
    if jc == ic:
     atemp.append('**'+xc+'**')
    else:
     atemp.append(xc)
   atempx = ' '.join(atemp)
   print(atempx)
   print()
 parts.append(prev)
 #print 'line=',line.encode('utf-8')
 for ic,c in enumerate(parts):
  #print ic,c
  if (len(c) == 1) and (ord(c) <= 127):
   # skip ascii character character
   continue
  if c not in asdict:
   asdict[c] = 0
  asdict[c] = asdict[c] + 1
 #exit(0)

def check_ea(filein,fileout):
# set up regex callback 'repl' with access to dictionary asdict
 asdict = {}
 # read the lines of the file
 f = codecs.open(filein,encoding='utf-8',mode='r')
 n = 0
 for line in f:
  line = line.rstrip()
  n = n + 1
  update_asdict(line,asdict)

 f.close()
 keys = asdict.keys()
 print(n,"lines in",filein)
 print(len(keys),"extended ascii codes found in",filein)

 keys = sorted(keys)
 print( len(keys))
 outlines = []
 for key in keys:
  asobj = asdict[key]
  #key1=convert(key)
  # key is a string
  # ords = ["\u%04x" % ord(c) for c in key]
  ords = []
  for c in key:
   try:
    uval = r"\u%04x" % ord(c)
   except:
    print('WARNING: Cannot convert: c=%s, ord(c)=%s' % (c,ord(c)))
    uval = "???"
   ords.append(uval)
  ordstr = ''.join(ords)
  names = [unicodedata.name(c) for c in key]
  namestr = ' + '.join(names)
  #out = "%s  (\\u%04x) %5d := %s" %(key,ord(key),asobj,namestr)
  out = "%s  (%s) %5d := %s" %(key,ordstr,asobj,namestr)
  outlines.append(out)
 fout = codecs.open(fileout,'w','utf-8')
 for out in outlines:
  fout.write("%s\n" % out)
 fout.close()
#-----------------------------------------------------
if __name__=="__main__":
 filein = sys.argv[1]
 fileout = sys.argv[2]
 check_ea(filein,fileout)


import sys
import datetime, time
from pyspark.sql import functions as F
from pyspark.sql.types import StructType, StructField, StringType, IntegerType

def get_null_perc(spark, df, null_cols):
    """ Get null/empty percentage for columns

    Args:
        spark (Spark): SparkSession object
        df (DataFrame): dataframe to perform null/empty analysis on
        null_cols (List): list of columns that need to be considered for analysis

    Returns:
        DataFrame: dataframe with null check analysis
    """
    schema = StructType([ \
        StructField("Column",StringType(),True), \
        StructField("NullPercentage",StringType(),True)
    ])
    emptyRDD = spark.sparkContext.emptyRDD()
    resultdf = spark.createDataFrame(emptyRDD, schema=schema)

    df_count = df.count()
    for x in null_cols:
        if x.upper() in (name.upper() for name in df.columns):
            df_null_count = df.select(F.col(x)).filter(F.col(x).isNull() | (F.col(x) == '')).count()
            df_null = spark.createDataFrame([[x, str(df_null_count*100.0/df_count) + '%' ]],schema=schema)
            resultdf = resultdf.union(df_null)
    return resultdf

def get_summary_numeric(df, numeric_cols):
    """ Get Summary for numeric columns

    Args:
        df (DataFrame): dataframe to perform analysis on
        numeric_cols (List): list of columns that need to be considered for analysis

    Returns:
        DataFrame: dataframe with summary analysis
    """
    for x in numeric_cols:
        if x.upper() not in (name.upper() for name in df.columns):
            numeric_cols.remove(x)
    return df.select(numeric_cols).summary()

def get_distinct_counts(spark, df, aggregate_cols):
    """ Get distinct count for columns

    Args:
        spark (Spark): SparkSession object
        df (DataFrame): dataframe to perform distinct count analysis on
        aggregate_cols (List): list of columns that need to be considered for analysis

    Returns:
        DataFrame: dataframe with distinct count analysis
    """
    schema = StructType([ \
        StructField("Column",StringType(),True), \
        StructField("DistinctCount",StringType(),True)
    ])

    emptyRDD = spark.sparkContext.emptyRDD()
    resultdf = spark.createDataFrame(emptyRDD, schema=schema)

    for x in aggregate_cols:
        if x.upper() in (name.upper() for name in df.columns):
            df_distinct_count = df.select(F.col(x)).distinct().count()
            df_distinct = spark.createDataFrame([[x, str(df_distinct_count)]],schema=schema)
            resultdf = resultdf.union(df_distinct)

    return resultdf

def get_distribution_counts(spark, df, aggregate_cols):
    """ Get Distribution Counts for columns

    Args:
        spark (Spark): SparkSession object
        df (DataFrame): dataframe to perform null/empty analysis on
        aggregate_cols (List): list of columns that need to be considered for analysis

    Returns:
        Array: Array of objects with dataframes
    """
    result = []
    for x in aggregate_cols:
        if x.upper() in (name.upper() for name in df.columns):
            result.append(df.groupby(F.col(x)).count().sort(F.col("count").desc()))
    ###

    return result

def get_mismatch_perc(spark, df, data_quality_cols_regex):
    """ Get Mismatch Percentage for columns

    Args:
        spark (Spark): SparkSession object
        df (DataFrame): dataframe to perform null/empty analysis on
        data_quality_cols_regex (Dictionary): Dictionary of columns/regex-expression for data quality analysis

    Returns:
        DataFrame: DataFrame with data quality analysis
    """
    schema = StructType([ \
        StructField("Column",StringType(),True), \
        StructField("MismatchPercentage",StringType(),True)
    ])

    emptyRDD = spark.sparkContext.emptyRDD()
    resultdf = spark.createDataFrame(emptyRDD, schema=schema)

    df_count = df.count()
    for key, value in data_quality_cols_regex.items():
        if key.upper() in (name.upper() for name in df.columns):
            df_regex_not_like_count = df.select(F.col(key)).filter(~F.col(key).rlike(value)).count()
            df_regex_not_like = spark.createDataFrame([[key, str(df_regex_not_like_count*100.0/df_count) + '%']],schema=schema)
            resultdf = resultdf.union(df_regex_not_like)

    return resultdf


titulo = 'Promoção Otica'
'''
Data: 07/03/2023
Autor: Felipe
Descrição: Faz uma promoção de óculos
de acordo com a idade
'''
print('Bem vindo ao programa ' + titulo)

print('-'*10 + 'OTICAS OLHO BOM' + '-'*10)

valor_do_oculos = 200

print('Promoção especial no óculos de R$' +
      str(valor_do_oculos) + ',00')
print('Informe sua idade para saber o desconto')
idade = int(input())

desconto = idade

if desconto > 80:
    desconto = 80
elif desconto < 10:
    desconto = 10

desconto_em_reais = desconto * valor_do_oculos/100

valor_final = valor_do_oculos - desconto_em_reais

print('Você tem ' + str(idade) + ' anos e ganhou um \
desconto de ' + str(desconto) + '%')
print('O óculos sairá por apenas ' + str(valor_final) +
      ' reais')

2 - Estrutura Condicional/Lista_2_EX_4.py


from __future__ import absolute_import
import abc

import pytest
from django.core.exceptions import ValidationError
from django.utils import timezone
from nose.tools import assert_equal, assert_in, assert_raises

import mock

from framework.auth import Auth

from tests.base import DbTestCase
from osf_tests.factories import UserFactory, CommentFactory, ProjectFactory, PreprintFactory, RegistrationFactory, AuthUserFactory
from osf.models import NotableDomain, SpamStatus
from website import settings, mails

@pytest.mark.django_db
@mock.patch('framework.auth.views.mails.send_mail')
def test_throttled_autoban(mock_mail):
    settings.SPAM_THROTTLE_AUTOBAN = True
    user = AuthUserFactory()
    projects = []
    for _ in range(7):
        proj = ProjectFactory(creator=user)
        proj.flag_spam()
        proj.save()
        projects.append(proj)
    mock_mail.assert_called_with(osf_support_email=settings.OSF_SUPPORT_EMAIL,
            can_change_preferences=False,
            to_addr=user.username,
            user=user,
            mail=mails.SPAM_USER_BANNED)
    user.reload()
    assert user.is_disabled
    for project in projects:
        assert not project.is_public

@pytest.mark.enable_implicit_clean
class TestReportAbuse(DbTestCase):

    def setUp(self):
        super(TestReportAbuse, self).setUp()
        self.comment = CommentFactory()
        self.auth = Auth(user=self.comment.user)

    def test_report_abuse(self):
        user = UserFactory()
        time = timezone.now()
        self.comment.report_abuse(
            user, date=time, category='spam', text='ads', save=True)
        assert_equal(self.comment.spam_status, SpamStatus.FLAGGED)
        equivalent = dict(
            date=time,
            category='spam',
            text='ads',
            retracted=False
        )
        assert_in(user._id, self.comment.reports)
        assert_equal(self.comment.reports[user._id], equivalent)

    def test_report_abuse_own_comment(self):
        with assert_raises(ValueError):
            self.comment.report_abuse(
                self.auth.user,
                category='spam', text='ads',
                save=True
            )
        assert_equal(self.comment.spam_status, SpamStatus.UNKNOWN)

    def test_retract_report(self):
        user = UserFactory()
        time = timezone.now()
        self.comment.report_abuse(
            user, date=time, category='spam', text='ads', save=True
        )
        assert_equal(self.comment.spam_status, SpamStatus.FLAGGED)
        self.comment.retract_report(user, save=True)
        assert_equal(self.comment.spam_status, SpamStatus.UNKNOWN)
        equivalent = {
            'date': time,
            'category': 'spam',
            'text': 'ads',
            'retracted': True
        }
        assert_in(user._id, self.comment.reports)
        assert_equal(self.comment.reports[user._id], equivalent)

    def test_retract_report_not_reporter(self):
        reporter = UserFactory()
        non_reporter = UserFactory()
        self.comment.report_abuse(
            reporter, category='spam', text='ads', save=True
        )
        with assert_raises(ValueError):
            self.comment.retract_report(non_reporter, save=True)
        assert_equal(self.comment.spam_status, SpamStatus.FLAGGED)

    def test_retract_one_report_of_many(self):
        user_1 = UserFactory()
        user_2 = UserFactory()
        time = timezone.now()
        self.comment.report_abuse(
            user_1, date=time, category='spam', text='ads', save=True
        )
        assert_equal(self.comment.spam_status, SpamStatus.FLAGGED)
        self.comment.report_abuse(
            user_2, date=time, category='spam', text='all', save=True
        )
        self.comment.retract_report(user_1, save=True)
        equivalent = {
            'date': time,
            'category': 'spam',
            'text': 'ads',
            'retracted': True
        }
        assert_in(user_1._id, self.comment.reports)
        assert_equal(self.comment.reports[user_1._id], equivalent)
        assert_equal(self.comment.spam_status, SpamStatus.FLAGGED)

    def test_cannot_remove_flag_not_retracted(self):
        user = UserFactory()
        self.comment.report_abuse(
            user, category='spam', text='ads', save=True
        )
        self.comment.remove_flag(save=True)
        assert_equal(self.comment.spam_status, SpamStatus.FLAGGED)

    def test_remove_flag(self):
        self.comment.flag_spam()
        self.comment.save()
        assert_equal(self.comment.spam_status, SpamStatus.FLAGGED)
        self.comment.remove_flag(save=True)
        assert_equal(self.comment.spam_status, SpamStatus.UNKNOWN)

    def test_validate_reports_bad_key(self):
        self.comment.reports[None] = {'category': 'spam', 'text': 'ads'}
        with assert_raises(ValidationError):
            self.comment.save()

    def test_validate_reports_bad_type(self):
        self.comment.reports[self.auth.user._id] = 'not a dict'
        with assert_raises(ValidationError):
            self.comment.save()

    def test_validate_reports_bad_value(self):
        self.comment.reports[self.auth.user._id] = {'foo': 'bar'}
        with assert_raises(ValidationError):
            self.comment.save()

@pytest.mark.django_db
class TestSpamState:
    @pytest.fixture(params=[
        CommentFactory,
        ProjectFactory,
        PreprintFactory,
        RegistrationFactory,
        UserFactory,
    ])
    def spammable_thing(self, request):
        spammable_factory = request.param
        return spammable_factory()

    def test_flag_spam(self, spammable_thing):
        assert not spammable_thing.is_spammy
        assert not spammable_thing.is_spam
        spammable_thing.flag_spam()
        spammable_thing.save()
        assert spammable_thing.is_spammy
        assert not spammable_thing.is_spam

    def test_confirm_ham(self, spammable_thing):
        spammable_thing.confirm_ham(save=True)
        assert spammable_thing.is_ham

    def test_confirm_spam(self, spammable_thing):
        spammable_thing.confirm_spam(save=True)
        assert spammable_thing.is_spam

    @pytest.mark.parametrize('assume_ham', (True, False))
    @pytest.mark.parametrize('spam_status, expected_props', (
            (SpamStatus.UNKNOWN, {
                'is_spam': False,
                'is_spammy': False,
                'is_ham': False,
                'is_hammy': None,  # set in the test body based on assume_ham
            }),
            (SpamStatus.FLAGGED, {
                'is_spam': False,
                'is_spammy': True,
                'is_ham': False,
                'is_hammy': False,
            }),
            (SpamStatus.SPAM, {
                'is_spam': True,
                'is_spammy': True,
                'is_ham': False,
                'is_hammy': False,
            }),
            (SpamStatus.HAM, {
                'is_spam': False,
                'is_spammy': False,
                'is_ham': True,
                'is_hammy': True,
            }),
    ))
    def test_spam_status_properties(self, spammable_thing, assume_ham, spam_status, expected_props):
        if spam_status == SpamStatus.UNKNOWN:
            expected_props['is_hammy'] = assume_ham

        with mock.patch.object(type(spammable_thing), 'is_assumed_ham', new_callable=mock.PropertyMock) as mock_assumed_ham:
            mock_assumed_ham.return_value = assume_ham
            spammable_thing.spam_status = spam_status

            assert spammable_thing.is_spam == expected_props['is_spam']
            assert spammable_thing.is_spammy == expected_props['is_spammy']
            assert spammable_thing.is_ham == expected_props['is_ham']
            assert spammable_thing.is_hammy == expected_props['is_hammy']

@pytest.mark.django_db
class TestSpamCheckEmailDomain:
    @mock.patch('osf.models.spam.SpamMixin.do_check_spam', return_value=False)
    @mock.patch.object(settings, 'SPAM_SERVICES_ENABLED', True)
    @mock.patch.object(settings, 'SPAM_CHECK_PUBLIC_ONLY', False)
    def test_trusted_domain(self, mock_do_check_spam):
        user = UserFactory()
        project = ProjectFactory()

        # spam check should normally call do_check_spam
        assert not mock_do_check_spam.called
        is_spam = project.check_spam(user, saved_fields={'title'}, request_headers={})
        assert not is_spam
        assert mock_do_check_spam.called

        # but what if we trust the user's email domain?
        user_email_address = user.emails.values_list('address', flat=True).first()
        user_email_domain = user_email_address.rpartition('@')[2].lower()
        NotableDomain.objects.create(
            domain=user_email_domain,
            note=NotableDomain.Note.ASSUME_HAM_UNTIL_REPORTED,
        )

        # should not call do_check_spam this time
        mock_do_check_spam.reset_mock()
        assert not mock_do_check_spam.called
        is_spam = project.check_spam(user, saved_fields={'title'}, request_headers={})
        assert not is_spam
        assert not mock_do_check_spam.called


import jieba
import random
import argparse
from tqdm import tqdm
# import logging
# import pandas as pd

from confusion_set import PinyinConfusionSet, StrokeConfusionSet, YaclcConfusionSet
from utils import is_chinese_char

class yaclcAug:
    def __init__(self, yaclc_char_prob,
                 yaclc_word_prob,
                 replace_prob,
                 py_prob,
                 jy_prob,
                 sk_prob,
                 vocab_path='confusions/vocab.txt',
                 yaclc_char_file='confusions/confusion_char.txt',
                 yaclc_word_file='confusions/confusion_word.txt',
                 same_py_file='confusions/same_pinyin.txt',
                 simi_py_file='confusions/simi_pinyin.txt',
                 stroke_file='confusions/same_stroke.txt'):

        self.yaclc_word_prob = yaclc_word_prob
        self.yaclc_char_prob = yaclc_char_prob

        self.replace_prob = replace_prob
        self.py_prob_thr = py_prob
        self.jy_prob_thr = jy_prob + self.py_prob_thr
        self.sk_prob_thr = sk_prob + self.jy_prob_thr

        self.chinese_vocab = self.get_chinese_vocab(vocab_path)

        self.yaclc_char_confu = YaclcConfusionSet(yaclc_char_file)
        self.yaclc_word_confu = YaclcConfusionSet(yaclc_word_file)
        self.yaclc_char_keys = self.yaclc_char_confu.confusion.keys()
        self.yaclc_word_keys = self.yaclc_word_confu.confusion.keys()

        self.same_py_confu = PinyinConfusionSet(same_py_file)
        self.simi_py_confu = PinyinConfusionSet(simi_py_file)
        self.sk_confu = StrokeConfusionSet(stroke_file)

    def get_chinese_vocab(self, vocab_path):
        chinese_vocab = set()
        with open(vocab_path, 'r', encoding='utf-8') as f:
            for line in f.readlines():
                line = line.strip()
                if is_chinese_char(line):
                    chinese_vocab.add(line)
        return list(chinese_vocab)

    def word_replace(self, sent_list, word_cut, replace_indices):
        idx = 0
        for word in word_cut:
            if word in self.yaclc_word_keys:
                if random.random() <= self.yaclc_word_prob:
                    new_word = self.yaclc_word_confu.get_confusion_item(word)
                    if new_word is not None:
                        sent_list[idx: idx + len(word)] = list(new_word)
                        replace_indices[idx: idx + len(word)] = [1 for _ in range(len(word))]
            idx += len(word)

    def char_replace(self, sent_list, replace_indices):
        for idx in range(len(sent_list)):
            if replace_indices[idx] == 1:
                continue
            if sent_list[idx] in self.yaclc_char_keys and random.random() <= self.yaclc_char_prob:
                sent_list[idx] = self.yaclc_char_confu.get_confusion_item(sent_list[idx])
                replace_indices[idx] = 1

    def process(self, sent):
        length = len(sent)
        sent_list = list(sent)
        replace_indices = [0 for _ in range(length)]
        word_cut = jieba.lcut(sent)
        self.word_replace(sent_list, word_cut, replace_indices)
        self.char_replace(sent_list, replace_indices)

        for idx in range(length):
            if is_chinese_char(sent_list[idx]):
                if random.random() <= self.replace_prob:
                    prob = random.random()
                    if prob <= self.py_prob_thr:
                        new_char = self.same_py_confu.get_confusion_item(sent_list[idx])
                    elif prob <= self.jy_prob_thr:
                        new_char = self.simi_py_confu.get_confusion_item(sent_list[idx])
                    elif prob <= self.sk_prob_thr:
                        new_char = self.sk_confu.get_confusion_item(sent_list[idx])
                    else:
                        new_char = self.chinese_vocab[random.randint(0, len(self.chinese_vocab) - 1)]
                    if new_char is not None:
                        sent_list[idx] = new_char
                        replace_indices[idx] = 1

        if any(replace_indices):
            return ''.join(sent_list)
        return None

if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    parser.add_argument('-i', '--input_path',
                        help='Path to the source file',
                        required=True)
    parser.add_argument('-o', '--output_path',
                        help='Path to the target file',
                        required=True)
    args = parser.parse_args()

    aug = yaclcAug(yaclc_char_prob=0.1, yaclc_word_prob=0.01, replace_prob=0.01, py_prob=0.5, jy_prob=0.2, sk_prob=0.3)

    text_list = []
    with open(args.input_path, 'r', encoding='utf-8') as rfd:
        for line in rfd.readlines():
            line = line.strip().split('\t')[1].replace(' ', '')
            text_list.append(line)

    aug_text_list = []
    for text in tqdm(text_list, total=len(text_list)):
        new_text = aug.process(text)
        if new_text:
            aug_text_list.append((new_text, text))

    with open(args.output_path, 'w', encoding='utf-8') as wfd:
        for aug_text in aug_text_list:
            wfd.write(f'{(aug_text[0])}\t{aug_text[1]}\n')