import struct
import os
import time
import sys
import binascii
import numpy as np
import pickle

from lz78 import compression_lz78, decompression_lz78
from lz77 import compression_lz77, decompression_lz77
from huffman import compress_huffman, decompress_huffman

def add_header(new_contents, counter):
    pcd = f"""\
# .PCD v0.7 - Point Cloud Data file format
VERSION 0.7
FIELDS x y z intensity
SIZE 4 4 4 4
TYPE F F F F
COUNT 1 1 1 1
WIDTH {counter}
HEIGHT 1
VIEWPOINT 0 0 0 1 0 0 0
POINTS {counter}
DATA ascii
"""

    # Append data
    for line in new_contents:
        pcd += f"{line}"
    #1print("New contents: " + str(pcd[:3000]))
    return pcd

def remove_zeros(data):
    lines = data.split("\n")

    # iterate over the lines and remove any that contain "0 0 0 0"
    new_lines = [line for line in lines if "0 0 0 0" not in line]
    new = []
    counter = 0
    for line in new_lines:
        if line != "" and line[0].isdigit() and line != "#":
            counter += 1
            # separate the string by space
            line = line.split(" ")
            # keep only two digits after the decimal point and remove square brackets and quotes
            line = [f"{float(x):.2f}" if i < len(line)-1 else str(int(float(x))) for i, x in enumerate(line)]
            #line = [f"{float(x):.2f}" for x in line]
            new.append(" ".join(line))
    # join the remaining lines back together
    pcd = "\n".join(new)
    return pcd, counter

def cal_differences(lines, counter):
    # Split the lines into a list of strings by the newline character
    new_lines = lines.split("\n")
    # Initialize an empty list to store the data
    data = []
    # Iterate through each line
    for line in new_lines:
        # Check if the line is not empty, starts with a digit, and is not a comment
        if line != "" and line[0].isdigit() and line != "#":
            # Split the line by spaces and append it to the data list
            line = line.split(" ")
            data.append(line)
    # Initialize an empty list to store the result
    result = []
    # Initialize a counter for the row number
    counter_row = 0
    # Initialize a variable to store the previous values
    prev_values = None
    # Iterate through each row of data
    for row in data:
        # Initialize an empty list to store the current values
        curr_values = []
        # Check if the row has 4 values
        if len(row) == 4:
            # Increment the row counter
            counter_row += 1
            # Iterate through each value in the row
            for i, val in enumerate(row):
                # Check if there are previous values and the current index is within bounds
                if prev_values is not None and i <= len(row) - 1:
                    # Calculate the difference between the current value and the previous value
                    diff = (float(val)) - (float(prev_values[i]))
                    # Round the difference to 2 decimal places and append it to the current values list
                    curr_values.append(round(diff, 2))
                else:
                    # If there are no previous values or the index is out of bounds, append the current value as is
                    curr_values.append(float(val))
            # Set the previous values to the current row
            prev_values = row
            # Convert the last value of the current values list to an integer
            curr_values[3] = int(curr_values[3])
            # Append the current values to the result list
            result.append(curr_values)
            # Increment the row counter
            counter_row += 1
    # Create a temporary list to store the formatted rows
    tmp = []
    # Iterate through each row in the result list
    for row in result:
        # Join the values in the row with spaces and append it to the temporary list
        tmp.append(' '.join([str(val) for val in row]))
    # Join the rows in the temporary list with newline characters and assign it to the variable "pcd"
    pcd = "\n".join(tmp)
    # Return the formatted PCD data
    return pcd

def retrieve_original_data(lines):
    # Split the lines into a list of strings by the newline character
    new_lines = lines.split("\n")
    # Initialize an empty list to store the data
    data = []
    # Iterate through each line
    for line in new_lines:
        # Check if the line is not empty and not a comment
        if line != "" and line != "#":
            # Split the line by spaces and append it to the data list
            line = line.split(" ")
            data.append(line)

    # Initialize an empty list to store the result
    result = []
    # Initialize a variable to store the previous values
    prev_values = None
    # Initialize a counter for the row number
    counter_row = 0
    # Iterate through each row of data
    for row in data:
        # Initialize an empty list to store the current values
        curr_values = []
        # Iterate through each value in the row
        for i, val in enumerate(row):
            # Check if there are previous values and the current index is within bounds
            if prev_values is not None and i <= len(row) - 1:
                # Calculate the original data based on the previous value and the current value
                if float(val) > 0:
                    original_data = float(val) + float(prev_values[i])
                else:
                    original_data = float(prev_values[i]) - abs(float(val))
                # Round the original data to 2 decimal places and append it to the current values list
                curr_values.append(round(original_data, 2))
            else:
                # If there are no previous values or the index is out of bounds, append the current value as is
                curr_values.append(round(float(val), 2))
        curr_values[3] = int(curr_values[3])
        # Set the previous values to the current values
        prev_values = curr_values
        # Append the current values to the result list
        result.append(curr_values)
        # Increment the row counter
        counter_row += 1
    # Create a temporary list to store the formatted rows
    tmp = []
    # Iterate through each row in the result list
    for row in result:
        # Join the values in the row with spaces and append it to the temporary list
        tmp.append(' '.join([str(val) for val in row]))
    # Join the rows in the temporary list with newline characters and assign it to the variable "original_data"
    original_data = "\n".join(tmp)
    # Add a header to the original data using the number of rows in the result list
    original_data = add_header(original_data, len(result))
    # Return the formatted original data
    return original_data

def compression(to_compress, compressed, compression_algorithm):
    # Create the compressed folder if it doesn't already exist
    if not os.path.exists(compressed):
        os.makedirs(compressed)
    print("Compressing files...")

    # Iterate through all files in the to_compress folder
    for filename in os.listdir(to_compress):
        # Measure time of compression
        start = time.time()
        print(filename)
        # Get the size of the original file
        original_size = os.path.getsize(os.path.join(to_compress, filename))
        print("Original size: " + str(original_size) + " bytes")
        if filename.endswith(".pcd"):
            with open(os.path.join(to_compress, filename), "r") as f:
                # Read in the contents of the PCD file
                contents = f.read()
                # Remove any lines that contain "0 0 0 0" and get the counter for removed lines
                contents, counter = remove_zeros(contents)
                # Calculate differences between consecutive lines
                contents = cal_differences(contents, counter)
                new_size = None
                # Check the selected compression algorithm
                if compression_algorithm == "lz78":
                    # Compress the contents using LZ78
                    compressed_contents = compression_lz78(contents)
                    filename = filename[:-4]
                    with open(os.path.join(compressed, filename + "." + compression_algorithm), "w") as f2:
                        compressed_contents = ' '.join([f'{code}:{char}' for code, char in compressed_contents])
                        f2.write(compressed_contents)
                    new_size = os.path.getsize(os.path.join(compressed, filename + "." + compression_algorithm))

                elif compression_algorithm == "lz77":
                    # Compress the contents using LZ77
                    compressed_contents = compression_lz77(contents)
                    filename = filename[:-4]
                    with open(os.path.join(compressed, filename + "." + compression_algorithm), "w") as f2:
                        f2.write(str(compressed_contents))
                    new_size = os.path.getsize(os.path.join(compressed, filename + "." + compression_algorithm))

                elif compression_algorithm == "huff":
                    # Compress the contents using Huffman coding
                    compressed_contents, huffman_tree = compress_huffman(contents)
                    # Save the Huffman tree to a file
                    with open('huffman_tree.pickle', 'wb') as file:
                        pickle.dump(huffman_tree, file)
                    filename = filename[:-4]
                    with open(os.path.join(compressed, filename + "." + compression_algorithm), "w") as f2:
                        f2.write(str(compressed_contents))
                    new_size = os.path.getsize(os.path.join(compressed, filename + "." + compression_algorithm))

                elif compression_algorithm == "hul8":
                    # Compress the contents using Huffman coding and LZ78
                    compressed_contents, huffman_tree = compress_huffman(contents)
                    # Save the Huffman tree to a file
                    with open('huffman_tree_78.pickle', 'wb') as file:
                        pickle.dump(huffman_tree, file)
                    compressed_contents = compression_lz78(compressed_contents)
                    filename = filename[:-4]
                    with open(os.path.join(compressed, filename + "." + compression_algorithm), "w") as f2:
                        compressed_contents = ' '.join([f'{code}:{char}' for code, char in compressed_contents])
                        f2.write(compressed_contents)
                    new_size = os.path.getsize(os.path.join(compressed, filename + "." + compression_algorithm))

                elif compression_algorithm == "hul7":
                    # Compress the contents using Huffman coding and LZ77
                    compressed_contents, huffman_tree = compress_huffman(contents)
                    # Save the Huffman tree to a file
                    with open('huffman_tree_77.pickle', 'wb') as file:
                        pickle.dump(huffman_tree, file)
                    compressed_contents = compression_lz77(compressed_contents)
                    filename = filename[:-4]
                    with open(os.path.join(compressed, filename + "." + compression_algorithm), "w") as f2:
                        f2.write(str(compressed_contents))
                    new_size = os.path.getsize(os.path.join(compressed, filename + "." + compression_algorithm))

                print("New size: " + str(new_size) + " bytes")
        end = time.time()
        print("Compression time: " + str(end - start))
        break
    print("Done compressing files.")
    return counter

def decompression(to_decompress, decompressed, compression_algorithm):
    # Create the decompressed folder if it doesn't already exist
    if not os.path.exists(decompressed):
        os.makedirs(decompressed)
    print("Decompressing files...")
    # Iterate through all files in the to_decompress folder
    for filename in os.listdir(to_decompress):
        # Measure time of compression
        start = time.time()
        print(filename)
        # Get the size of the original file
        original_size = os.path.getsize(os.path.join(to_decompress, filename))
        print("Original size: " + str(original_size) + " bytes")
        if filename.endswith("." + compression_algorithm):
            with open(os.path.join(to_decompress, filename), "r") as f:
                # Read in the contents of the compressed file
                compressed_string = f.read().strip()
                # Parse the compressed data into a list of tuples

                if compression_algorithm == "lz78":
                    # Decompress the data using LZ78
                    compressed_data = []
                    for pair in compressed_string.split():
                        pair_list = pair.split(":")
                        if len(pair_list) == 2:
                            code, char = pair_list
                            compressed_data.append((int(code), char))
                    decompressed_contents = decompression_lz78(compressed_data)

                elif compression_algorithm == "lz77":
                    # Decompress the data using LZ77
                    decompressed_contents = decompression_lz77(compressed_string)

                elif compression_algorithm == "huff":
                    # Decompress the data using Huffman
                    with open('huffman_tree.pickle', 'rb') as file:
                        huffman_tree = pickle.load(file)
                    decompressed_contents = decompress_huffman(compressed_string, huffman_tree)

                elif compression_algorithm == "hul8":
                    # Decompress the data using Huffman and LZ78
                    compressed_data = []
                    for pair in compressed_string.split():
                        pair_list = pair.split(":")
                        if len(pair_list) == 2:
                            code, char = pair_list
                            compressed_data.append((int(code), char))
                    with open('huffman_tree.pickle', 'rb') as file:
                        huffman_tree = pickle.load(file)
                    decompressed_contents = decompression_lz78(compressed_data)
                    decompressed_contents = decompress_huffman(decompressed_contents, huffman_tree)

                # Retrieve the original data from the decompressed contents
                decompressed_contents = retrieve_original_data(decompressed_contents)

                filename = filename[:-5]
                with open(os.path.join(decompressed, filename + ".pcd"), "w") as f2:
                    f2.write(decompressed_contents)
                new_size = os.path.getsize(os.path.join(decompressed, filename + ".pcd"))
                print("New size: " + str(new_size) + " bytes")
        else:
            print("File is not compressed with " + compression_algorithm)
            continue
        end = time.time()
        print("Decompression time: " + str(end - start))
    print("Done decompressing files.")

def main():
    # Read in arg comp or decomp from command line to determine whether to compress or decompress
    action = sys.argv[1]
    if len(sys.argv) == 3:
        algorithm = sys.argv[2]
    else:
        algorithm = ""

    print("Action: " + action, " with Algorithm: " + algorithm)
    to_compress = "./to_compress_1000/"
    compressed = "./compressed_1000/"
    decompressed = "./decompressed_1000/"
    if action == "comp":
        print("Compressing files in " + to_compress + " and saving to " + compressed)
        # Call the compression function
        compression(to_compress, compressed, algorithm)
    elif action == "decomp":
        print("Decompressing files in " + compressed + " and saving to " + decompressed)
        # Call the decompression function
        decompression(compressed, decompressed, algorithm)

if __name__ == "__main__":
    # Call the main function
    main()


"""
    Module for the Neural Network model we use in TF 2.0
        Tf 2.0 is still in beta and far from being perfectly documented. We found additional insights here: https://pgaleone.eu/tensorflow/tf.function/2019/03/21/dissecting-tf-function-part-1/.
"""

import tensorflow as tf
from simulOfBioNN.nnUtils.chemTemplateNN.chemTemplateLayers import chemTemplateLayer
# from simulOfBioNN.nnUtils.chemTemplateNN.chemTemplateCpLayer import chemTemplateCpLayer

class chemTemplateNNModel(tf.keras.Model):
    """
        Implement the model we derived from our mathematical analysis at equilibrium of a CRN implemented with the template model.
        We make two assumption:
            First we neglect variations of templates concentrations. This assumption makes our model biased
            Secondly, at backpropagation we neglect the derivative of cp with respect to either a template concentration or a node concentration.
                (TODO: prove this second assumption can be made)
                This second assumption makes our training heuristic less correct but far more easy to implement.
        We provide support for the simple model:
            exonuclease reactions model at order 1, polymerase and nickase are grouped under the same enzyme.
    """
    def __init__(self, nbUnits, sparsities, reactionConstants,enzymeInitC,activTempInitC, inhibTempInitC, randomConstantParameter=None, usingLog = True, usingSoftmax = True ):
        """
            Initialization of the model:
                    Here we simply add layers, they remain to be built once the input shape is known.
                    Next steps are made at the build level, when the input shape becomes usable.
            Be careful: we here consider the system to have been rescaled by using T0 and C0 constants.
        :param nbUnits: list, for each layer its number of units
        :param sparsities: the value of sparsity in each layer
                """
        super(chemTemplateNNModel,self).__init__(name="",dtype=tf.float64)
        nbLayers = len(nbUnits)

        if tf.test.is_gpu_available():
            print("computing on GPU")
            Deviceidx = "/gpu:0"
        else:
            Deviceidx = "/cpu:0"
        # Test of the inputs
        assert len(sparsities)==len(nbUnits)

        self.layerList = []
        for e in range(nbLayers):
            self.layerList += [chemTemplateLayer(Deviceidx,units=nbUnits[e],sparsity=sparsities[e],dynamic=False,usingLog=usingLog,dtype=tf.float64)]

        self.reactionConstants = reactionConstants
        self.enzymeInitC = enzymeInitC
        self.activTempInitC = activTempInitC
        self.inhibTempInitC = inhibTempInitC
        self.randomConstantParameter =randomConstantParameter

        assert len(self.reactionConstants) == 11
        assert type(self.enzymeInitC) == float
        assert type(self.activTempInitC) == float
        assert type(self.inhibTempInitC) == float
        if self.randomConstantParameter is not None:
            assert type(self.randomConstantParameter) == tuple
            #TODO: implement randomized constants

        # If the log mode is activated:
        #   - initial inputs should be provided as log(value)
        #   - each layer will gives the log(value at equilibrium) as output instead of "value at equilibrium"
        self.usingLog = usingLog
        self.usingSoftmax = usingSoftmax

        self.writer = tf.summary.create_file_writer("tfOUT")
        self.writer.set_as_default()

        self.built = False
        self.gathered = None

    def build(self,input_shape):
        """
            Build the model and make the following initialization steps:
                1) Intermediate layer use a random heuristic to obtain a sparse initialization
                2) Based on the initialized topology, we can rescale our system.
                        We diminish the concentration of template.
                        We increase the concentration of enzyme.
                3) We then provide each layer with initialization values for their real weights.
                    The kernel of each layer is based on weights in {-1,0,1},
                    but at computation time we multiply these weights by a value describing the chemistry.
                4) If the random constant parameter is provided (default to None), we use a gaussian noise to randomise:
                        1) the concentration of template.
                        2) the constant parameters for each reactions.
        :param input_shape:
        :param reactionConstants: standard value for the constants
        :param enzymeInitC: concentration value for the enzyme that proved to be working well for a small neuron.
                             Will be rescaled to compensate for larger number.
        :param activTempInitC: concentration value for the activating template that proved to be working well for a small neuron.
                                Will be rescaled to compensate for larger number.
        :param inhibTempInitC: concentration value for the inhibiting template that proved to be working well for a small neuron.
                                Will be rescaled to compensate for larger number.
        :param randomConstantParameter: randomConstantParameter: tuple of size 2 of list, default to None. First value is used for template (list of size 2),
                                        Second value for reactions constant (list of size reaction constants).
        :return:
        """
        self.rescaleFactor = tf.Variable(0,trainable=False,dtype=tf.float64)

        # We need to Initiate variable for the call to the herited build which will compile the call and thus need it.
        # BUT at the time of this function the model is still in eager mode, and the graph activated in the non-eager mode doesn't get all informations
        # THUS this instantation shall be made in non-eager mode, and the only moment to catch it is within the call function (see call).
        #self.non_eager_building(input_shape)

        modelsConstantShape =[(input_shape[-1],self.layerList[0].units)]+[(l.units,self.layerList[idx+1].units) for idx,l in enumerate(self.layerList[:-1])]
        input_shapes = [input_shape]+[(input_shape[0],l.units) for l in self.layerList[:-1]]
        for idx,l in enumerate(self.layerList):
            l.build(input_shapes[idx])

        self.rescaleFactor.assign(tf.keras.backend.sum([l.get_rescaleFactor() for l in self.layerList], axis=-1))
        for l in self.layerList:
            l.set_constants(self.reactionConstants,self.enzymeInitC,self.activTempInitC,self.inhibTempInitC,self.rescaleFactor)

        self.meanGatheredCps = tf.Variable(0,dtype=tf.float64,trainable=False)
        self.built = True
        self.mycps = tf.Variable(tf.zeros(1,dtype=tf.float64),trainable=False)
        super(chemTemplateNNModel,self).build(input_shape)

        print("model successfully built")

    def force_rescale(self,rescaleFactor):
        """
            Force the rescale factor to take the input value instead of the number of nodes in the graph.
        :param rescaleFactor: rescale value
        :return:
        """
        if not self.built:
            raise Exception("model should be built before calling this function")
        for l in self.layerList:
            l.rescale(rescaleFactor)
        self.rescaleFactor.assign(rescaleFactor)

    def greedy_set_cps(self,inputs):
        inputs = tf.convert_to_tensor(inputs,dtype=tf.float64)
        cps = self.obtainCp(inputs)
        tf.print(cps)
        tf.print(tf.nn.moments(cps,axes=[0]))
        self.mycps.assign([tf.reduce_mean(cps)])

    @tf.function
    def call(self, inputs, training=None, mask=None):
        """
            Call function for out chemical model.
                At training time we verify whether the model has changed total number of template due to backpropagation.
                    In this case we compute new values for the rescale factor, and update in all template the real weights.
        :param inputs: tensor for the inputs, if of rank >= 2, we need to split it.
        :param training: if we are in training or inference mode (we save time of checking model change)
        :param mask: we don't use it here...
        :return:
        """
        """
            We would like to proceed with a batching point of view.
            The problem here, is that tf.map_fn creates a graph for each realisation, making us loose the initialization on the current graph...
            Thus we cannot use it here, while this has not been fixed in tensorflow!
        """
        inputs = tf.cast(tf.convert_to_tensor(inputs),dtype=tf.float64)

        if training:
            self.verifyMask()
        inputs = inputs/self.rescaleFactor

        if self.usingLog:
            inputs = tf.exp(inputs)

        gatheredCps = tf.stop_gradient(tf.fill([tf.shape(inputs)[0]],tf.reshape(self._obtainCp(inputs[0]),())))
        gatheredCps = tf.reshape(gatheredCps,((tf.shape(inputs)[0],1)))
        tf.assert_equal(tf.shape(gatheredCps),(tf.shape(inputs)[0],1))
        #
        # gatheredCps = tf.stop_gradient(self.obtainCp(inputs))
        # gatheredCps = tf.fill([tf.shape(inputs)[0]],tf.reshape(self.mycps,()))
        # gatheredCps = tf.reshape(gatheredCps,((tf.shape(inputs)[0],1)))

        #self.meanGatheredCps.assign(tf.reduce_mean(gatheredCps))
        #tf.summary.scalar("mean_cp",data=tf.reduce_mean(gatheredCps),step=tf.summary.experimental.get_step())

        x = self.layerList[0](inputs,cps=gatheredCps,isFirstLayer=True)
        for l in self.layerList[1:]:
            if self.usingLog:
                x = l(tf.exp(x),cps=gatheredCps)
            else:
                x = l(x,cps=gatheredCps)
        if self.usingSoftmax:
            if self.usingLog:
                s = tf.keras.activations.softmax(tf.exp(x))
            else:
                s = tf.keras.activations.softmax(x)
        else:
            s = x
        return s

    def predConcentration(self, inputs, layerObserved,nodeObserved):
        inputs = tf.convert_to_tensor(inputs,dtype=tf.float64)
        tf.print(inputs.shape,"inputs shape")
        tf.assert_rank(inputs,2)
        tf.print("rescale factor for TF:",self.rescaleFactor)
        inputs = inputs/self.rescaleFactor
        gatheredCps = tf.stop_gradient(self.obtainCp(inputs))
        #self.meanGatheredCps.assign(tf.reduce_mean(gatheredCps))
        #tf.summary.scalar("mean_cp",data=tf.reduce_mean(gatheredCps),step=tf.summary.experimental.get_step())
        tf.print(inputs[0]," first input after rescale")
        x = self.layerList[0](inputs,cps=gatheredCps,isFirstLayer=True)
        if layerObserved==0:
            tf.print("outputs shape:",x.shape)
            return x[:,nodeObserved],gatheredCps
        tf.print(x,"x")
        tf.print(gatheredCps,"gatheredcps")
        for idx,l in enumerate(self.layerList[1:]):
            x = l(x,cps=gatheredCps)
            tf.print(x,"x")
            tf.print(idx,"idx")
            if tf.equal(idx,layerObserved-1):
                return x[:,nodeObserved],gatheredCps

    @tf.function
    def obtainCp(self,inputs):
        gatheredCps = tf.map_fn(self._obtainCp,inputs,parallel_iterations=32,back_prop=False)

        return gatheredCps

    @tf.function
    def _obtainCp(self,input):

        if(input.shape.rank<2):
            input = tf.reshape(input,(1,tf.shape(input)[0]))
        #first we obtain the born sup:

        bornsupIncremental = tf.cast(tf.fill([1],1.),dtype=tf.float64)
        bornsupcpdiff = self._computeCPdiff(bornsupIncremental,input)
        #tf.print("ended initial bornsup with 1 :",bornsupIncremental," found value",bornsupcpdiff)
        #tf.print("starting while")
        for idx in tf.range(20): #stop after 20 iteration
            bornsupIncremental = bornsupIncremental * 10.
            bornsupcpdiff = self._computeCPdiff(bornsupIncremental,input)
            if tf.greater(bornsupcpdiff[0],0.):
                #tf.print("ended bornsup loop",idx," found value",bornsupcpdiff, "using as sup:",bornsupIncremental)
                break
            #tf.print("ended bornsup loop",idx," found value",bornsupcpdiff)
            # if tf.equal(idx,20):
            #
                # bornsupcp0,x = self.layerList[0].layer_cp_born_sup(input)
                # layercp = tf.fill([1],1.)
                # for l in self.layerList[1:]:
                #     layercp,x = l.layer_cp_born_sup(x)
                #     bornsupcp0 += layercp
                # #tf.print("had to use layer born sup and found",bornsupcp0)
                # bornsupIncremental = tf.reshape(bornsupcp0,bornsupIncremental.shape)

        cpmin = tf.cast(tf.fill([1],1.),dtype=tf.float64)
        cp = self.brentq(self._computeCPdiff,cpmin,bornsupIncremental,input)
        return cp

    @tf.function
    def _computeCPdiff(self,cp,input):
        new_cp = tf.cast(tf.fill([1],1.),dtype =tf.float64)
        layercp,x = self.layerList[0].layer_cp_equilibrium(cp,input,isFirstLayer=True)
        new_cp += layercp
        for l in self.layerList[1:]:
            layercp,x = l.layer_cp_equilibrium(cp,x)
            new_cp += layercp
        new_cp = new_cp-cp
        return (-1)*new_cp

    def measureModelRelevancy(self,input):
        inputs = tf.stack([input])
        if(input.shape.rank<2):
            input = tf.reshape(input,(1,tf.shape(input)[0]))
        cp = self.obtainCp(inputs)
        Inhib = []
        cp2kd = []
        x,b,c=self.layerList[0].get_inhib_and_output(cp,input,isFirstLayer=True)
        Inhib += [b]
        cp2kd += [c]
        for l in self.layerList[1:]:
            x,b,c=self.layerList[0].get_inhib_and_output(cp,x,isFirstLayer=True)
            Inhib += [b]
            cp2kd += [c]
        return Inhib,cp2kd

    def lamda_computeCPdiff(self,input):
        if(input.shape.rank<2):
            input = tf.reshape(input,(1,tf.shape(input)[0]))
        return lambda cp: self._computeCPdiff(cp,input)

    @tf.function
    def getFunctionStyle(self,cpArray,X0):
        """
            Given an input, compute the value of f(cp) for all cp in cpArray (where f is the function we would like to find the roots with brentq)
        :param cpArray:
        :param X0:
        :return:
        """
        cpArray=tf.cast(tf.convert_to_tensor(cpArray),dtype=tf.float64)
        X0 = tf.cast(tf.convert_to_tensor(X0),dtype=tf.float64)
        func = self.lamda_computeCPdiff(X0)
        gatheredCps = tf.map_fn(func,cpArray,parallel_iterations=32,back_prop=False)
        return gatheredCps

    @tf.function
    def verifyMask(self):
        newRescaleFactor = tf.stop_gradient(tf.keras.backend.sum([l.get_rescaleFactor() for l in self.layerList], axis=-1))
        if(not tf.equal(self.rescaleFactor,newRescaleFactor)):
            self.rescaleFactor.assign(newRescaleFactor)
            for l in self.layerList:
                l.rescale(self.rescaleFactor)

    def updateArchitecture(self,masks,reactionsConstants,enzymeInitC,activTempInitC,inhibTempInitC):
        for idx,m in enumerate(masks):
            self.layerList[idx].set_mask(tf.convert_to_tensor(m,dtype=tf.float64))

        self.reactionConstants = reactionsConstants
        self.enzymeInitC = enzymeInitC
        self.activTempInitC = activTempInitC
        self.inhibTempInitC = inhibTempInitC

        self.rescaleFactor.assign(tf.keras.backend.sum([l.get_rescaleFactor() for l in self.layerList], axis=-1))
        for l in self.layerList:
            l.set_constants(self.reactionConstants,self.enzymeInitC,self.activTempInitC,self.inhibTempInitC,self.rescaleFactor)

    def logCp(self,epoch,logs=None):
        cp = self.meanGatheredCps
        tf.summary.scalar('mean cps', data= cp, step=epoch)
        return cp

    @tf.function
    def brentq(self, f, xa, xb,args=(),xtol=tf.constant(10**(-12),dtype=tf.float64), rtol=tf.constant(4.4408920985006262*10**(-16),dtype=tf.float64),iter=tf.constant(100)):
        xpre = tf.cast(tf.fill([1],0.),dtype=tf.float64) + xa
        xcur = tf.cast(tf.fill([1],0.),dtype=tf.float64)+ xb
        xblk = tf.cast(tf.fill([1],0.),dtype=tf.float64)
        fblk = tf.cast(tf.fill([1],0.),dtype=tf.float64)
        spre = tf.cast(tf.fill([1],0.),dtype=tf.float64)
        scur = tf.cast(tf.fill([1],0.),dtype=tf.float64)
        fpre = f(xpre, args)
        fcur = f(xcur, args)

        if tf.equal(fpre[0],0):
            return xpre
        if tf.equal(fcur[0],0):
            return xcur
        else:
            tf.assert_less((fpre*fcur)[0],tf.cast(0,dtype=tf.float64),message="seems like fpre*fcur have same sign",summarize=1)
            tf.assert_greater(fcur,tf.cast(0,dtype=tf.float64))

        for i in tf.range(iter):
            if tf.less((fpre*fcur)[0],0):
                xblk = xpre
                fblk = fpre
                spre = xcur - xpre
                scur = xcur - xpre
            if tf.less(tf.abs(fblk)[0],tf.abs(fcur)[0]):
                xpre = xcur
                xcur = xblk
                xblk = xpre

                fpre = fcur
                fcur = fblk
                fblk = fpre

            delta = (xtol + rtol*tf.abs(xcur))/2
            sbis = (xblk - xcur)/2
            if tf.equal(fcur[0],0) or tf.less(tf.abs(sbis)[0],delta[0]):
                break #BREAK FAILS HERE!!! ==> strange behavior?
            else:
                if tf.greater(tf.abs(spre)[0],delta[0]) and tf.less(tf.abs(fcur)[0],tf.abs(fpre)[0]):
                    if tf.equal(xpre[0],xblk[0]):
                        # /* interpolate */
                        stry = -fcur*(xcur - xpre)/(fcur - fpre)
                    else :
                        # /* extrapolate */
                        dpre = (fpre - fcur)/(xpre - xcur)
                        dblk = (fblk - fcur)/(xblk - xcur)
                        stry = -fcur*(fblk*dblk - fpre*dpre)/(dblk*dpre*(fblk - fpre))

                    mymin = tf.minimum(tf.abs(spre), 3*tf.abs(sbis) - delta) #Here would not understand...
                    spre=tf.where(tf.less(2*tf.abs(stry)-mymin,0),scur,sbis)
                    scur=tf.where(tf.less(2*tf.abs(stry)-mymin,0),stry,sbis)
                else:
                    # /* bisect */
                    spre = sbis
                    scur = sbis
                xpre = xcur
                fpre = fcur
                if tf.greater(tf.abs(scur)[0],delta[0]):
                    xcur += scur
                else:
                    if tf.greater(sbis[0],0):
                        xcur += delta
                    else:
                        xcur += -delta
            fcur = f(xcur, args)
        return xcur


import pandas as pd
import urllib
csv = pd.read_csv("https://docs.google.com/spreadsheets/d/10T3lIOc5fZgsyvmYeWOlmgsaRiGI2BhYNBPY5YH0EFk/export?format=csv&id=10T3lIOc5fZgsyvmYeWOlmgsaRiGI2BhYNBPY5YH0EFk&gid=0")
csv.dropna()
csv.drop_duplicates()
for ind in csv.index:
    try:
        print(str(csv["titles"][ind]))

        urllib.request.urlretrieve("https://i.ytimg.com/vi/"+str(csv["id"][ind])+"/hqdefault.jpg", "c:/Users/Skull/Documents/github/Thumbnail-clickbait-detection/data_manualy_tag/"+str(csv["total"][ind])+"_"+str(csv["id"][ind])+".jpg")
    except Exception as e:
        print("error on :"+str(csv["titles"][ind]))


# -*- coding: utf-8 -*-

from app import app
from flask import render_template
from app.models import Todo, TodoForm, NameForm
# 这个包还得导入,日了蛋
from flask import request, current_app, make_response, redirect, abort, jsonify, session, url_for
# 导入 bootstrap
from flask_bootstrap import Bootstrap
# 导入 Moment( 格式化时间 )
from flask_moment import Moment
from datetime import datetime
import json
from flask_mail import Mail

mail = Mail(app)
bootstrap = Bootstrap(app)
moment = Moment(app)

@app.route('/form', methods=['GET','POST'])
def form():
	name = None
	form = NameForm()
	if form.validate_on_submit():
		session['name'] = form.name.data
		return redirect(url_for('form'))
	return render_template('form.html', name = session.get('name'), form = form)

@app.route('/json')
def jsonSomething():
	'''返回 json'''
	a = [{"name" : "value", "haha" : "lalal" },{'name' : 'fuck'}]
	return jsonify(a)

@app.route('/time')
def time():
	return render_template('user.html', current_time = datetime.utcnow())

# 动态路由
@app.route('/user/<name>')
def user(name):
	# 打印请求头
	# print(request.headers)

	# 程序上下文
	# print(current_app.name)

	# 程序所有的路由
	# print(app.url_map)

	# return '
hello, %s
' % name

	# 第二个参数是 http 状态码
	# return '我是404哦', 404

	# 还可以返回 response
	# response = make_response('
hello, %s
' % name)
	# response.set_cookie('key','value')
	# return response

	# 重定向
	# return redirect('http://www.baidu.com')

	# abort 不会把控制权交还给调用它的函数，而是抛出异常把控制权交给 Web 服务器
	# abort(404)

	return render_template('user.html',name=name)

@app.errorhandler(404)
def page_not_found(e):
	return render_template('404.html'), 404

@app.errorhandler(500)
def page_not_found(e):
	return render_template('500.html'), 500

@app.route('/')
def index():
	form = TodoForm()
	todos = Todo.objects.all()
	return render_template('index.html', todos=todos, form=form)

@app.route('/add', methods=['POST',])
def add():
	form = TodoForm(request.form)
	if form.validate_on_submit():
		content = request.form.get("content")
		todo = Todo(content=content)
		todo.save()
	todos = Todo.objects.all()
	return render_template('index.html', todos=todos, form=form)

@app.route('/done/<string:todo_id>', methods=['POST',])
def done(todo_id):
	todo = Todo.objects.get_or_404(id=todo_id)
	todo.status = 1
	todo.save()
	todos = Todo.objects.all()
	return render_template('index.html',todos=todos)

@app.route('/undone/<string:todo_id>', methods=['POST',])
def undone(todo_id):
	todo = Todo.objects.get_or_404(id=todo_id)
	todo.status = 0
	todo.save()
	todos = Todo.objects.all()
	return render_template('index.html',todos=todos)

@app.route('/delete/<string:todo_id>')
def delete(todo_id):
	todo = Todo.objects.get_or_404(id=todo_id)
	todo.delete()
	todos = Todo.objects.all()
	return render_template('index.html', todos=todos)


name=["张三","李四","王二"]
num=["01","02","03"]
score=[90,80,70]

k=["姓名","学号","成绩"]

st1=dict.fromkeys(k)
print(st1)

st=[st1,st1.copy(),st1.copy()]
print(st)

for i in st:
    n=st.index(i)
    i["姓名"]=name[n]
    i["学号"]=num[n]
    i["成绩"]=score[n]

print(st)

Python/数据达人培养计划/6、Python映射：字典dict/课时47、dict字典的常用操作.py

课时47、dict字典的常用操作.py


class Solution:
    def minTaps(self, n: int, ranges: List[int]) -> int:
        jump=[0]*(n+1)
        for i in range(len(ranges)):
            left=max(0,i-ranges[i])
            right=min(n,ranges[i]+i)
            jump[left]=max(jump[left],right-left)
        print(jump)
        ans=start=end=0
        for i in range(len(jump)-1):
            start=max(start,i+jump[i])
            if start==i and jump[i]==0:
                return -1
            if end==i:
                end=start
                ans+=1
        return ans


import plotly.express as px
import streamlit as st
import pandas as pd
from datetime import datetime as dt
import matplotlib
import matplotlib.pyplot as plt

def color_metric_background(metric_value, color_map, metric_min = 0,metric_max=1 ):
    # Define the colormap
    cmap = plt.get_cmap(color_map)

    # Normalize the metric value to the range [0, 1]
    normalized_value = (metric_value - metric_min) / (metric_max - metric_min)

    # Map the normalized value to the colormap range [0, 256]
    cmap_value = int(normalized_value * 255)

    # Get the RGB color tuple from the colormap
    color = cmap(cmap_value)[:3]

    # Convert RGB color to hexadecimal
    hex_color = '#%02x%02x%02x' % tuple([int(x * 255) for x in color])

    # Create CSS style for the colored background
    background_style = f'background-color: {hex_color};'

    return background_style

def make_plots(df, y_bar,y_line, x) :
    fig = px.bar(
        df,
        x=resolution,
        y=y_bar,
    )
    fig_available = px.line(
        df,
        x=resolution,
        y=y_line,
        labels=y_line, color_discrete_sequence=['red']
    )

    fig.add_traces(
    list(fig_available.select_traces())

    )

    name = ['avg used','available']

    for i in range(len(fig.data)):
        fig.data[i]['name'] = name[i]
        fig.data[i]['showlegend'] = True
    return fig

def get_kpi_metrics(df, metrics):
    kpi = []
    for metric in metrics:
        kpi_col = f'kpi_{metric}'
        usage_col = f'usage_{metric}'
        available_col =f'available_{metric}'
        df[kpi_col] = round(100*df[usage_col]/df[available_col],1)
        kpi.append(kpi_col)
    return df, kpi

# Load data
df_metrics = pd.read_csv("data/input_data.csv")
df_metrics["month"] = pd.PeriodIndex(df_metrics.week, freq="M")
df_metrics["quarter"] = pd.PeriodIndex(df_metrics.week, freq="Q")

# Inputs
metrics= ['A','B']
COLOR_MAP = 'RdYlGn'

# Calculate avg metrics by month
df_metrics_month = (
    df_metrics.groupby(["month", "quarter"])
    .agg(
        usage_A=("usage_A", "mean"),
        usage_B=("usage_B", "mean"),
        available_A =("available_A", "mean"),
        available_B=("available_B", "mean"),
    )
    .reset_index()
)

# Calculate avg metrics by quarter
df_metrics_quarter = (
    df_metrics.groupby(["quarter"])
    .agg(
        usage_A=("usage_A", "mean"),
        usage_B=("usage_B", "mean"),
        available_A =("available_A", "mean"),
        available_B=("available_B", "mean"),
    )
    .reset_index()
)

# Store all resolutions in a dictionary
## Having the information pre-aggregated facilitates the speed of the application
resolutions = ["week","month", "quarter"]
metric_frames = {"week":df_metrics,"month": df_metrics_month, "quarter": df_metrics_quarter}

# Calculate KPI metrics by week, quarter and month for all metrics in list
df_metrics, kpi_metrics  = get_kpi_metrics(df_metrics, metrics)
df_metrics_month, kpi_metrics  = get_kpi_metrics(df_metrics_month, metrics)
df_metrics_quarter, kpi_metrics = get_kpi_metrics(df_metrics_quarter, metrics)

# Create a table to display in the dashboard.
## The table is transposed to ease reading
kpi_frames = {"week" : df_metrics[['week'] + kpi_metrics].set_index('week').T,
              "month": df_metrics_month[['month'] + kpi_metrics].set_index('month').T,
              "quarter": df_metrics_quarter[['quarter'] + kpi_metrics].set_index('quarter').T}

st.set_page_config(layout="wide")
col1, col2 = st.columns(2)
col1.title("KPI usage Dashboard")
col2.markdown("[Link to repo](https://github.com/alexane-rose/streamlit-app)")

# Current Month metrics
cols = st.columns(len(metrics))
current_month = pd.Period(dt.today(),freq='M')
current_month_metrics = df_metrics_month.loc[df_metrics_month.month == current_month ].reset_index(drop=True)

i=0

for col in cols:
   kpi_col = f'kpi_{metrics[i]}'
   col.markdown(f'
{current_month_metrics[kpi_col][0]}%
', unsafe_allow_html=True)
   i+=1

resolution = st.radio("Time resolution", resolutions)

# KPI table
st.table(kpi_frames[resolution].style.background_gradient(cmap=COLOR_MAP, vmin= 0 , vmax=100).format("{:.1f}%"))

# Bar charts
df_metrics_quarter['quarter'] = df_metrics_quarter['quarter'].astype('str')
df_metrics_month['month'] = df_metrics_month['month'].astype('str')

figs = []
for metric in metrics:
    usage_col = f'usage_{metric}'
    available_col = f'available_{metric}'
    fig = make_plots(metric_frames[resolution],usage_col,available_col,resolution)
    figs.append(fig)

cols = st.columns(len(metrics))
i=0
for col in cols:
    col.markdown(f'### Avg Usage vs Availble of {metrics[i]} over time')
    col.plotly_chart(figs[i],  theme="streamlit", use_container_width=True)
    i+=1


import json
import boto3
import time

def lambda_handler(event, context):
    list_num = event["list_num"]
    with open("prefix_list_part" + str(list_num) + ".txt") as words_file:
        words = json.load(words_file)
        print(words)
        for word in words:
            time.sleep(6)
            client = boto3.client('lambda')
            response = client.invoke(
                FunctionName='KohlsScraper',
                InvocationType="Event",
                LogType="None",
                Payload= """{"toScrape": \"""" + word + "\"}"
            )
    if (event["linked"]):
        if list_num < 16:
            time.sleep(200)
            client.invoke(
                FunctionName='RunOrchestrator',
                InvocationType="Event",
                LogType="None",
                Payload= "{\"list_num\": "+ str(list_num + 1) + ",\"linked\": true}"
            )
    return {
        'statusCode': 200,
        'body': json.dumps('Hello from Lambda!')
    }


# Import modules
import os
import csv

# Set path for file
csvpath = os.path.join('Resources',"02-Homework_03-Python_Instructions_PyPoll_Resources_election_data.csv")

# Set values, lists, dictionaries
candidates_list = []
candidates_votes = {}
winner_max_value = 0

# Open the CSV reader
with open(csvpath,'r') as csvfile:
    csvreader = csv.reader(csvfile, delimiter=",")
    next(csvreader)

    #Converting csvreader to list to refer
    voter_data = list(csvreader)

    # Calculate the total number of votes cast - NEED WORK
    total_votes = len(list(voter_data))

    # Iterate over rows
    for row in voter_data:

    # Compile a complete list of candidates who received votes
        if row[2] not in candidates_list:
            candidates_list.append(row[2])

        # Calculate the total number of votes each candidate won
        if row[2] not in candidates_votes:
            candidates_votes[row[2]] = 0

        candidates_votes[row[2]] += 1

# Print Summary Part One
print("Election Results")
print("----------------------")
print(f"Total Votes: {total_votes}")
print("----------------------")

# Calculate the percentage of votes each candidate won
for name in candidates_list:
    count = candidates_votes[name]
    pct = 100*(count / total_votes)
    candidate_summary = (f"{name}: {format(pct,'.3f')}% ({count})")
    print(candidate_summary)

# Find the winner of the election based on popular vote.
    if winner_max_value < count:
        winner_max_value = count
        winner_name = name

print("----------------------")
print(f"Winner: {winner_name}")
print("----------------------")

# Write the above to a text file
with open("Analysis/pypollmain.txt", "w") as f:

    f.write("Election Results\n")
    f.write("----------------------\n")
    f.write(f"Total Votes: {total_votes}\n")
    f.write("----------------------\n")
    for name in candidates_list:
        count = candidates_votes[name]
        pct = 100*(count / total_votes)
        candidate_summary = (f"{name}: {format(pct,'.3f')}% ({count})")
        f.write(f"{candidate_summary}]\n")
    f.write("----------------------\n")
    f.write(f"Winner: {winner_name}\n")
    f.write("----------------------\n")


import os
import json
import datetime
import requests

from django.shortcuts import render, get_object_or_404, redirect, reverse
from django.contrib.auth.models import User
from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin
from django.views.generic import ListView, DetailView, CreateView, DeleteView
from django.contrib.auth.decorators import login_required
from django.contrib import messages
from django.http import HttpResponseRedirect
from django.utils.crypto import get_random_string

from django.utils.decorators import method_decorator

from .models import Post, SearchQuery, GlobalQuery
from .forms import SearchKeywordQueryForm, GlobalSearchForm, PostForm
from .services import get_account_info, search_transcript, get_similar_words, search_transcript_sequence, \
                    search_across_files, check_file_and_get_length, json_to_diarized_txt, json_to_diarized_txt_list
from .services import create_wordcloud
from .decode_service import decode_audio, save_api_response

max_word_len = 100
max_context_len = 10

class HomeView(ListView):
    model = Post
    template_name = 'decoding_app/home.html'

@method_decorator([login_required], name='dispatch')
class UserPostListView(LoginRequiredMixin, ListView):
    '''
    List view class which renders all posts by a user.
    '''

    model = Post
    template_name = 'decoding_app/user_posts.html'
    context_object_name = 'posts'
    ordering = ['-date_posted']
    paginate_by = 5

    def get_queryset(self):
        user = get_object_or_404(User, username=self.kwargs.get('username'))
        posts = Post.objects.filter(author=user)\
                            .filter(delete_after_transcription=False)\
                            .order_by('-date_posted')
        return posts

@method_decorator([login_required], name='dispatch')
class PostDetailView(LoginRequiredMixin, UserPassesTestMixin, DetailView):
    '''
    DetailView class which displays details of a specific audio file showing
    its transcription and allows the user to interact with the it.
    '''

    model = Post

    def get_context_data(self, **kwargs):
        '''
        Called apon loading the page. It will render the trascription, wordcloud and
        old search queries for a particular post.
        '''
        context = super(PostDetailView, self).get_context_data(**kwargs)

        # word stats
        if len(context['post'].words_json) > 0:
            context['word_stats'] = json.loads(context['post'].words_json)
            if context['post'].done_decoding:
                if len(context['post'].transcription) > 0:
                    context['wordcloud'] = create_wordcloud(context['post'].transcription, context['post'].language)

        # split transcription for readmore or less
        context['post'].transcription_less = ' '.join(context['post'].transcription.split()[:max_word_len])
        context['post'].transcription_more = ' '.join(context['post'].transcription.split()[max_word_len:])

        # add a form for search queries
        context['form'] = SearchKeywordQueryForm

        # load previous search queries
        context['searchqueries'] = SearchQuery.objects.filter(post_pk=context['post'].pk).order_by('-date_searched')[0:5]

        # query specific logic
        if len(context['searchqueries']) > 0:

            keyword = str(context['searchqueries'][0])
            context['keyword'] = keyword

            # search transcript
            if len(keyword.split()) == 1:
                res_c, res_l, res_r = search_transcript(keyword, context['post'].transcription, \
                                                json.loads(context['post'].words_json), max_context_len)
            else:
                res_c, res_l, res_r = search_transcript_sequence(keyword, context['post'].transcription, \
                                                                json.loads(context['post'].words_json), max_context_len)

            context['searchresult_stats_c'] = json.dumps(res_c)
            context['searchresult_stats_l'] = json.dumps(res_l)
            context['searchresult_stats_r'] = json.dumps(res_r)
            context['num_occurences'] = len(res_c)

        return context

    def post(self, request, slug):

        if request.is_ajax():

            keyword = str(request.POST.get('search_query')).lower()

            # find similar
            if request.POST.get('find_similar') == 'true':

                lang = Post.objects.filter(slug=slug)[0].language
                if len(keyword.split()) == 1:
                    similar_words = get_similar_words(keyword, lang)
                else:
                    # just take the first word for suggestions
                    similar_words = get_similar_words(keyword.split()[0], lang)

                context = {'suggestions': similar_words}
                return render(request, 'decoding_app/similar_words.html', context)

            # search results and save
            elif request.POST.get('search_results') == 'true':

                # save query
                post = Post.objects.filter(slug=slug)[0]
                SearchQuery.objects.create(keyword=keyword, post_pk=post.pk, author=request.user)

                # search transcript
                if len(keyword.split()) == 1:
                    res_c, res_l, res_r = search_transcript(keyword, post.transcription, \
                                                    json.loads(post.words_json), max_context_len)
                else:
                    res_c, res_l, res_r = search_transcript_sequence(keyword, post.transcription, \
                                                                    json.loads(post.words_json), max_context_len)

                context = {}
                context['searchresult_stats_c'] = json.dumps(res_c)
                context['searchresult_stats_l'] = json.dumps(res_l)
                context['searchresult_stats_r'] = json.dumps(res_r)
                context['num_occurences'] = len(res_c)

                context['keyword'] = keyword

                return render(request, 'decoding_app/search_results.html', context)

            # render search history div
            else:
                context = {}
                post = Post.objects.filter(slug=slug)[0]
                context['searchqueries'] = SearchQuery.objects.filter(post_pk=post.pk).order_by('-date_searched')[0:5]
                return render(request, 'decoding_app/last_searched.html', context)

        # post with django if coming from global search
        form = SearchKeywordQueryForm(request.POST)
        if form.is_valid():
            # save keyword
            obj = form.save(commit=False)
            obj.author = request.user
            obj.post_pk = pk
            obj.save()

            return HttpResponseRedirect(reverse('post-detail', kwargs={'slug': slug}))

    def test_func(self):
        post = self.get_object()
        if self.request.user == post.author:
            return True
        return False

    def form_valid(self, form):
        form.instance.author = self.request.user
        return super().form_valid(form)

@login_required
def post_create_view(request):

    init={'delete_after_transcription': request.user.profile.delete_after_transcription,\
                             'sampling_freq': request.user.profile.sampling_freq,
                             'language': request.user.profile.language,
                             'file_type': request.user.profile.file_type,
                             'automatic_punctuation': request.user.profile.automatic_punctuation,
                             'automatic_diarization': request.user.profile.automatic_diarization,
                            }
    context = {}
    form = PostForm(initial=init)
    context['form'] = form

    return render(request, "decoding_app/post_form.html", context)

class PostDeleteView(LoginRequiredMixin, UserPassesTestMixin, DeleteView):
    model = Post
    success_url = '/'

    def test_func(self):
        post = self.get_object()
        if self.request.user == post.author:
            return True
        return False

    def delete(self, request, *args, **kwargs):
        """
        Call the delete() method on the fetched object and then redirect to the
        success URL.
        """
        self.object = self.get_object()
        success_url = self.get_success_url()

        post = self.object

        dt = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        print(f"Info {dt}: Delete post user:{request.user} post:{post}")

        post.delete_after_transcription = True
        post.audio_file.delete()
        post.transcription = "Not saved"
        post.words_json = "Not saved"
        post.save()

        return redirect('home')

from django.views.generic import View
from django.http import HttpResponse
import csv
from docx import Document

class FileView(LoginRequiredMixin, View):
    '''
    View to serve files to download
    '''
    def get(self, request, *args, **kwargs):

        #post = Post.objects.filter(pk=kwargs['pk'])[0]
        post = Post.objects.filter(slug=kwargs['slug'])[0]
        # choose extension
        file_type = kwargs['file_type']
        # get the post name
        fn = os.path.splitext(post.title)[0]
        cd = f'attachment; filename="{fn}.{file_type}"'

        # mangage diarization
        if post.automatic_diarization:
            transcript = json_to_diarized_txt(json.loads(post.words_json))
        else:
            transcript = post.transcription

        if file_type == 'txt':
            response = HttpResponse(transcript, content_type='application/text')
        elif file_type == 'docx':

            document = Document()
            document.add_heading(fn, level=1)
            document.add_paragraph("")

            if post.automatic_diarization:
                transcript = json_to_diarized_txt_list(json.loads(post.words_json))
                for par in transcript:
                    document.add_paragraph(par)
            else:
                document.add_paragraph(transcript)

            response = HttpResponse(content_type='application/vnd.openxmlformats-officedocument.wordprocessingml.document')
            document.save(response)

        else:
            response = HttpResponse(post.words_json, content_type='application/json')

        response['Content-Disposition'] = cd

        return response

@login_required
def global_search(request):
    '''
    View that renders a form for the serach across files
    '''
    context = {}
    context['num_posts'] = Post.objects.filter(author=request.user).count()

    if request.method == 'POST':

        keyword = str(request.POST.get('keyword')).lower()

        # (1) redirect and search specific post
        if request.POST.get('global_search_redirect') == 'true':

            keyword = str(request.POST.get('keyword')).lower()

            pk = str(request.POST.get('post_id'))
            post = Post.objects.filter(pk=pk)[0]

            # create query before page loads
            SearchQuery.objects.create(keyword=keyword, post_pk=post.pk, author=request.user)
            return HttpResponseRedirect(reverse('post-detail', kwargs={'slug': post.slug}))

        # (2) perform search on given keyword
        else:
            # do global search
            form = GlobalSearchForm(request.POST)
            if form.is_valid():
                # save query
                obj = form.save(commit=False)
                obj.author = request.user
                obj.save()

                # get transcripts which contain keyword
                w = obj.keyword.lower()
                res_counts = search_across_files(w, request.user)
                context['results_keyword'] = w
                context['results'] = res_counts

    else:
        form = GlobalSearchForm()

    # load previous search queries and suggestions
    context['searchqueries'] = GlobalQuery.objects.filter(author=request.user).order_by('-date_searched')[0:5]

    if len(context['searchqueries']) > 0:
        keyword = str(context['searchqueries'][0])
        context['keyword'] = keyword

        # assume language is multi
        lang = 'mul'
        if len(keyword.split()) == 1:
            context['suggestions'] = get_similar_words(keyword, lang)
        else:
            context['suggestions'] = get_similar_words(keyword.split()[0], lang)

    context['form'] = form

    return render(request, 'decoding_app/global_search.html', context)

def about(request):
    context = {'title': 'About'}
    return render(request, 'decoding_app/about.html', context)

from django.views.decorators.csrf import csrf_exempt
from django.http import JsonResponse

def verify_and_upload(request):

    '''
    Before posting check that all files are valid and together it does not exceed limit.
    Support for uploading any media file (even video), support for uploading multiple files
    which we handle by sending individual requests once validated.
    '''

    if request.method == 'POST':

        files = [request.FILES[key] for key in request.FILES]
        numspeakers_select = request.POST.get('numspeakers_select', False)

        # check each file
        total_duration = 0
        file_lengths = []
        file_num_channels = []
        fnames = []
        fnames_response = ''
        # ----------------------------------------
        # (1) validation step
        # ----------------------------------------
        for f in files:

            f_name = f.name
            media, dur, num_channels = check_file_and_get_length(f)

            fnames_response += f'{f_name}, '
            fnames.append(f_name)

            dt = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
            print(f"Info {dt}: File is_media:{media} duration:{dur}")
            file_lengths.append(dur)
            file_num_channels.append(num_channels)
            total_duration += dur

            # redirect if any file is not media
            if media != 1:
                messages.warning(request, f'Problem with file "{f_name}". Are you sure file contains audio?')
                return JsonResponse({"message": "not media"})

        # check that the account has not exceeded free trial or not suspended
        if request.user.profile.total_seconds + total_duration > int(os.environ.get('FREE_TRIAL_SECONDS')):
            if request.user.profile.allow_decoding_overide != True:

                remain = int(os.environ.get('FREE_TRIAL_SECONDS')) - request.user.profile.total_seconds
                messages.warning(request, f'Upload exceeds trial. You have {remain}s left. Contact us to continue using this service.')
                return JsonResponse({"message": "expired"})

        # ----------------------------------------
        # (2) upload
        # ----------------------------------------

        messages.success(request, f"Busy transcribing {fnames_response[0:-2]}! \
                 You will get an email with the transcription or view it in your uploads.")

        # get metadata with post form
        form = PostForm(request.POST)

        for i, f in enumerate(files):

            # (2.1) Save entry
            if form.is_valid():
                obj = form.save(commit=False)

            # if NB and stereo assume one speaker per channel
            if obj.sampling_freq == 'NB' and file_num_channels[i] == 2:
                obj.automatic_diarization = True

            obj.title = fnames[i]
            obj.author = request.user
            obj.audio_file = f

            obj.audio_length = file_lengths[i]
            if obj.automatic_diarization:
                if numspeakers_select:
                    obj.num_speakers = numspeakers_select

            obj.audio_length = file_lengths[i]
            obj.slug = get_random_string(length=32)
            obj.save()

            # (2.2) begin decoding (use .delay to use task runner to manage async process)
            print(f"Info {dt}: Starting job:{obj.pk} {obj.slug}")
            decode_audio.delay(obj.pk)

        return JsonResponse({"message": f"{fnames_response[0:-2]} uploaded!"})

def contact(request):
    context = {'title': 'Contact Us',
               'email': 'info@saigen.co.za',
               'tel': '+27 (84) 951 9002',
               'address': ['35 Brickfield Rd', 'Brickfield Canvas', 'Salt River', 'Cape Town 7925'],
               'web_address': 'https://saigen.co.za'
               }

    return render(request, 'decoding_app/contact_us.html', context)

@login_required
def account(request):

    # get account info for current user
    num_files, num_seconds = get_account_info(request.user)

    context = {'title': 'Account Information',
               'numfiles': num_files,
               'numseconds': num_seconds,
               }

    return render(request, 'decoding_app/account_info.html', context)

def expired(request):
    context = {'title': 'Your free trial is up or your account is outstanding.'}
    context['body'] = "To continue using our service please contact us"
    return render(request, 'decoding_app/expired.html', context)

from django.http import HttpResponse, JsonResponse
from django.views.decorators.csrf import csrf_exempt
@csrf_exempt
def jobcomplete(request):

    dt = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    print(f"Info {dt}: Job complete {request.headers}")

    if request.method == 'GET':
        return HttpResponse('
Return jobs here
')

    elif request.method == 'POST':

        # validate
        # ----------------
        auth = request.headers.get("X_API_KEY", "")
        if auth != os.getenv('LAMBDA_API_KEY'):
            print(f"Info {dt}: Unauthorized ({auth})")
            return JsonResponse({'result': 'Error','message': 'Unauthorized'})

        user_id = request.headers.get("USER_ID", "")
        if len(user_id) > 0:
            if not User.objects.filter(pk=user_id).exists():
                print(f"Info {dt}: Unkown user ({user_id})")
                return JsonResponse({'result': 'Error','message': 'Unkown user'})
        else:
            print(f"Info {dt}: No user id")
            return JsonResponse({'result': 'Error', 'message': 'Specify USER_ID'})

        job_id = request.headers.get("JOB_ID", "")
        if len(job_id) > 0:
            if not Post.objects.filter(slug=job_id).exists():
                print(f"Info {dt}: Unkown job ({job_id})")
                return JsonResponse({'result': 'Error','message': 'Unkown job'})
        else:
            print(f"Info {dt}: No job id")
            return JsonResponse({'result': 'Error', 'message': 'Specify JOB_ID'})

        if Post.objects.filter(slug=job_id)[0].author != User.objects.filter(pk=user_id)[0]:
            print(f"Info {dt}: Job ({job_id}) does not belong to user ({user_id})")
            return JsonResponse({'result': 'Error','message': 'Incorrect job or user'})

        status = request.headers.get("STATUS", "")
        if status not in ['ERROR', 'SUCCESS']:
            print(f"Info {dt}: Incorrect status")
            return JsonResponse({'result': 'Error','message': 'Incorrect STATUS - Please return (ERROR | SUCCESS)'})

        # save
        # ----------
        print(f"Info {dt}: Saving response")
        s = request.FILES["file"].read()
        json_payload=json.loads(s)
        save_api_response(job_id, json_payload)

        return JsonResponse({'result': 'Success','message': 'Successfully recieved and recorded'}, status=200)


# -*- coding: utf-8 -*-
"""
@Author         :  LEITENG
@Version        :
------------------------------------
@File           :  作业(2.29).py
@Description    :
@CreateTime     :  2020/4/17 16:57
------------------------------------
@ModifyTime     :
"""
from functools import reduce

def main():
    '''
    作业1、用递归方式生成斐波拉契数列（用一个递归函数实现）
    作业2、用递归方式求n!
    作业3、格式化用户的英文名，要求首字母大小，其它字母小写
    作业4、将用户英文名、年龄、性别三个集合的数据结合到一起，形成一个集合
    作业5、过滤性别为男的用户
    作业6、求性别为男的用户的平均年龄
    :return:
    '''
    # 作业1、用递归方式生成斐波拉契数列（用一个递归函数实现）
    # print([fibb(i) for i in range(6)])

    # 作业2、用递归方式求n!
    # print(jie(5))

    # 作业3、格式化用户的英文名，要求首字母大小，其它字母小写
    # names = ['jOoei', 'ifIFJ', 'SIXdff', 'lOSDni']
    # names_filter = map(lambda item: item.lower().title(), names)
    # print(list(names_filter))

    # 作业4、将用户英文名、年龄、性别三个集合的数据结合到一起，形成一个集合
    name = ['a', 'b', 'c']
    age = [12, 15, 17]
    sex = ['man', 'fenal', 'man']
    info = list(map(lambda name, age, sex: (name, age, sex), name, age, sex))
    print(info)

    # 作业5、过滤性别为男的用户
    info_filter = list(filter(lambda item: item[2] == 'man', info))
    print(info_filter)

    # 作业6、求性别为男的用户的平均年龄
    age_sum = reduce(lambda x, y: x[1] + y[1], info_filter)
    print(age_sum/len(info_filter))

def jie(n):
    if n <= 1:
        return 1
    return n * jie(n-1)

def fibb(n):
    if n <= 1:
        return 1
    return fibb(n-1) + fibb(n-2)

if __name__ == '__main__':
    main()


import torch
import torch.nn as nn
import numpy as np
import torch.nn.functional as F
import torch.optim as optim
import os
import glob
import matplotlib.pyplot as plt
import cv2
# import Dataset calss
from torch.utils.data import Dataset

""" Convolutional block:
    It follows a two 3x3 convolutional layer, each followed by a batch normalization and a relu activation.
"""
class conv_block(nn.Module):
    def __init__(self, in_c, out_c):
        super().__init__()

        self.conv1 = nn.Conv2d(in_c, out_c, kernel_size=3, padding=1)
        self.bn1 = nn.BatchNorm2d(out_c)

        self.conv2 = nn.Conv2d(out_c, out_c, kernel_size=3, padding=1)
        self.bn2 = nn.BatchNorm2d(out_c)

        self.relu = nn.ReLU()

    def forward(self, inputs):
        x = self.conv1(inputs)
        x = self.bn1(x)
        x = self.relu(x)

        x = self.conv2(x)
        x = self.bn2(x)
        x = self.relu(x)

        return x

class encoder_block(nn.Module):
    def __init__(self, in_c, out_c):
        super().__init__()

        self.conv = conv_block(in_c, out_c)
        self.pool = nn.MaxPool2d(kernel_size=2, stride=2)

    def forward(self, inputs):
        x = self.conv(inputs)
        p = self.pool(x)

        return x, p

class decoder_block(nn.Module):
    def __init__(self, in_c, out_c):
        super().__init__()

        self.up = nn.ConvTranspose2d(in_c, out_c, kernel_size=2, stride=2)
        self.conv = conv_block(in_c, out_c)

    def forward(self, inputs, skip):
        x = self.up(inputs)
        x = torch.cat([x, skip], dim=1)
        x = self.conv(x)

        return x

class UNet(nn.Module):
    def __init__(self, in_c, out_c, num_filters=64):
        super(UNet, self).__init__()

        # Encoder
        self.encoder1 = encoder_block(in_c, num_filters)
        self.encoder2 = encoder_block(num_filters, num_filters * 2)
        self.encoder3 = encoder_block(num_filters * 2, num_filters * 4)
        self.encoder4 = encoder_block(num_filters * 4, num_filters * 8)

        # Bottleneck
        self.bottleneck = conv_block(num_filters * 8, num_filters * 16)

        # Decoder
        self.decoder1 = decoder_block(num_filters * 16, num_filters * 8)
        self.decoder2 = decoder_block(num_filters * 8, num_filters * 4)
        self.decoder3 = decoder_block(num_filters * 4, num_filters * 2)
        self.decoder4 = decoder_block(num_filters * 2, num_filters)

        # Classifier
        self.outputs = nn.Conv2d(num_filters, out_c, kernel_size=1)

    def forward(self, x):
        # Encoder
        enc1, x = self.encoder1(x)
        enc2, x = self.encoder2(x)
        enc3, x = self.encoder3(x)
        enc4, x = self.encoder4(x)

        # Bottleneck
        x = self.bottleneck(x)

        # Decoder
        x = self.decoder1(x, enc4)
        x = self.decoder2(x, enc3)
        x = self.decoder3(x, enc2)
        x = self.decoder4(x, enc1)

        # Classifier
        outputs = self.outputs(x)

        return outputs

# Inspired by: https://github.com/nikhilroxtomar/Semantic-Segmentation-Architecture/blob/main/PyTorch/unet.py


class Solution:
    def isPalindrome(self, s: str) -> bool:
        str = "".join([i for i in s if i.isalnum()]).lower()
        half = len(str)//2
        forward = str[:half]
        if len(str) % 2 == 0:
            backward = str[:half-1:-1]
        else:
            backward = str[:half:-1]
        print(f"{forward} \n{backward}")
        if forward == backward:
            return True
        return False

sol = Solution()
print(sol.isPalindrome(
    "A a"))


'''
Google Map Scraper Using Selenium
Code By Nikhil Rathour
17 July, 2022
'''

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
import csv
import time
from bs4 import BeautifulSoup as bs

pages = 2

header = ["title", "address", "website", "phone","profiles"]
data = []

options = webdriver.ChromeOptions()
options.add_experimental_option('excludeSwitches', ['enable-logging'])
driver = webdriver.Chrome(options=options)
driver.get('https://internshala.com/internship/detail/python-development-internship-in-hyderabad-at-zestiot-technologies-private-limited1674450367')

driver.implicitly_wait(5)

close = driver.find_element(By.ID,"no_thanks").click()
# details = driver.find_elements(By.LINK_TEXT, 'View details')

driver.implicitly_wait(5)

def find_elements():

            driver.implicitly_wait(10)

            try:
                profile = driver.find_element(By.CLASS_NAME,"profile")
                print("Profile: ",profile.text)
                company = driver.find_element(By.CLASS_NAME,"company_name")
                print("company: ",company.text)
                location = driver.find_element(By.ID,"location_names")
                print('Loacation: ', location.text)
                stipend = driver.find_element(By.CLASS_NAME, "stipend")
                print('stipend :', stipend.text)
                website = driver.find_element(By.LINK_TEXT,"Website")
                print('Website: ',website.get_attribute('href'))

            # try:
            #     stipend = soup.find('i', class_='')
            #     print('stipend: ',stipend.string)
            # except Exception as e:
            #     print(e)
            # try:
            #     website = soup.find('div', class_='')
            #     print('website: ', website)
            except Exception as e:
                 print(e)

find_elements()


from django.shortcuts import render, redirect
from .models import Note

def index(request):
    context={
        'notes' : Note.objects.all()
    }
    return render(request,'index.html',context)

def base(request,id):
    context={
        'note': Note.objects.get(id=id)
    }
    return render(request,'base.html',context)

def form(request):
    return render(request,'form.html')

def create(request):
    Name=request.GET['Name']
    Description = request.GET['Description']
    Note.objects.create(name=Name, description=Description)

    return redirect('form-page')

def update_form(request,id):
    context={
        'note': Note.objects.get(id=id)
    }

    return render(request,'updateform.html',context)

def update(request,id):
    Name=request.GET['Name']
    Description = request.GET['Description']
    note = Note.objects.get(id=id)
    note.name=Name
    note.description=Description
    note.save()

    return redirect('index-page')

def delete(request,id):
    Note.objects.get(id=id).delete()
    return redirect('index-page')


from cell import Cell
from firework import Firework

class Screen:
    turtles = []
    grid = []
    lines = 0
    cols = 0

    def __init__(self, lines, cols):
        self.lines = lines
        self.cols = cols
        self.grid = self.build_grid()
        self.turtles = []
        for _ in range(10):
            self.add_turtle(Firework(lines, cols))

    def __str__(self):
        s = ""
        for line in self.grid:
            for column in line:
                s += column.__str__()
            s += "\n"
        return s.strip()

    def build_grid(self):
        grid = []
        for _ in range(self.lines):
            inner = []
            grid.append(inner)
            for _ in range(self.cols):
                inner.append(Cell())
        return grid

    def add_turtle(self, turtle):
        self.turtles.append(turtle)

    def tick(self):
        turtles = []
        for turtle in self.turtles:
            if not turtle.finished():
                turtle.tick(self)
                turtles.append(turtle)
        self.turtles = turtles