import hashlib  # common interface to many different secure hash and message digest algorithms
import re  # provides regular expression matching operations

import lief  # cross platform library which able to parse, modify and abstract ELF, PE and MachO formats
import numpy as np  # the fundamental package for scientific computing with Python
from logzero import logger  # robust and effective logging for Python
from sklearn.feature_extraction import FeatureHasher  # implements feature hashing, aka the hashing trick

# get lief version
LIEF_MAJOR, LIEF_MINOR, _ = lief.__version__.split('.')
# disable lief logging
lief.logging.disable()

# check installed lief version capabilities
LIEF_EXPORT_OBJECT = int(LIEF_MAJOR) > 0 or (int(LIEF_MAJOR) == 0 and int(LIEF_MINOR) >= 10)
LIEF_HAS_SIGNATURE = int(LIEF_MAJOR) > 0 or (int(LIEF_MAJOR) == 0 and int(LIEF_MINOR) >= 11)

class FeatureType(object):
    """ Base class from which each feature type may inherit. """

    name = ''  # feature name
    dim = 0  # feature dimension

    def __repr__(self):  # get feature description (name + dim)
        """ Get unambiguous object representation in string format.

        Returns:
             Unambiguous object representation in string format.
        """

        return '{}({})'.format(self.name, self.dim)  # return formatted string

    def raw_features(self,
                     bytez,  # PE file binary data
                     lief_binary):  # lief parsing of PE file binaries
        """ Generate a JSON-able representation of the file (raw features).

        Args:
            bytez: PE file binary data
            lief_binary: Lief parsing of PE file binaries
        """

        raise NotImplementedError  # will be overridden by child classes

    def process_raw_features(self,
                             raw_obj):  # dictionary of raw features
        """ Generate a feature vector from the raw features.

        Args:
            raw_obj: Dictionary of raw features
        """

        raise NotImplementedError  # will be overridden by child classes

    def feature_vector(self,
                       bytez,  # PE file binary data
                       lief_binary):  # lief parsing of PE file binaries
        """ Directly calculate the feature vector from the sample itself. This should only be implemented differently
        if there are significant speedups to be gained from combining the two functions.

        Args:
            bytez: PE file binary data
            lief_binary: Lief parsing of PE file binaries
        Returns:
            Feature vector.
        """

        # get raw features from the sample; then generate feature vector
        return self.process_raw_features(self.raw_features(bytez, lief_binary))

class ByteHistogram(FeatureType):
    """ Byte histogram (count + non-normalized) over the entire binary file. """

    name = 'histogram'  # feature name
    dim = 256  # feature dimension

    def __init__(self):
        """ Initialize ByteHistogram class. """

        super(FeatureType, self).__init__()

    def raw_features(self,
                     bytez,  # PE file binary data
                     lief_binary):  # lief parsing of PE file binaries

        # interpret PE file binary data as a 1-dimensional array of (8-bit) unsigned integers,
        # then count the number of occurrences of each value in the array
        # (setting minimum number of bins for the output array to 256 to have always the same output size)
        # -> Each bin will give the number of occurrences of its index value in the original ndarray
        counts = np.bincount(np.frombuffer(bytez, dtype=np.uint8), minlength=256)

        # convert the numpy ndarray to a python list and return it
        return counts.tolist()

    def process_raw_features(self,
                             raw_obj):  # byte histogram raw features

        # create counts ndarray from the raw byte histogram interpreting the values as floats
        counts = np.array(raw_obj, dtype=np.float32)

        # sum all counts together
        counts_sum = counts.sum()

        # normalize counts dividing them by the total sum
        normalized = counts / counts_sum

        # return normalized histogram
        return normalized

class ByteEntropyHistogram(FeatureType):
    """ 2d byte/entropy histogram based loosely on (Saxe and Berlin, 2015).
    This roughly approximates the joint probability of byte value and local entropy.
    See Section 2.1.1 in https://arxiv.org/pdf/1508.03096.pdf for more info.
    """

    name = 'byteentropy'
    dim = 256

    def __init__(self,
                 step=1024,  # step size
                 window=2048):  # window size
        """ Initialize ByteEntropyHistogram class. """

        super(FeatureType, self).__init__()

        # set attributes
        self.window = window
        self.step = step

    def _entropy_bin_counts(self,
                            block):  # ndarray containing a piece (block) of the PE file binary data
        """ Get bin frequencies (counts) and entropy bin index (Hbin).

        Args:
            block: Ndarray containing a piece (block) of the PE file binary data
        Returns:
            Entropy bin index (Hbin) and bin frequencies (counts).
        """

        # calculate bin frequency:
        # shift block bytes to the right by 4 positions (dividing them by 16),
        # then count the number of occurrences of each value in the array
        # (setting minimum number of bins for the output array to 16)
        # in order to have a coarse histogram, with 16 bytes per bin
        c = np.bincount(block >> 4, minlength=16)  # 16-bin histogram

        # calculate bin probability:
        # get a copy of "c" ndarray, casting its values to float and then dividing them by the window size
        p = c.astype(np.float32) / self.window

        # get non-zero bins indexes (where(c) -> where c is not zero)
        wh = np.where(c)[0]

        # calculate entropy:
        # get p values where c is not zero, multiply them with the -log2 of themselves,
        # sum the results and then multiply by 2
        # (x2 because we reduced information by half: 256 bins (8 bits) to 16 bins (4 bits))
        H = np.sum(-p[wh] * np.log2(p[wh])) * 2

        # get the bin index where to store histogram "c", we have up to 16 bins (max entropy is 8 bits)
        Hbin = int(H * 2)
        if Hbin == 16:  # handle entropy = 8.0 bits
            Hbin = 15

        # return bin index "Hbin" and histogram "c"
        return Hbin, c

    def raw_features(self,
                     bytez,  # PE file binary data
                     lief_binary):  # lief parsing of PE file binaries

        # initialize output as a 2d grid of 16x16 zeros (ints)
        output = np.zeros((16, 16), dtype=np.int)

        # interpret PE file binary data as a 1-dimensional array of (8-bit) unsigned integers
        a = np.frombuffer(bytez, dtype=np.uint8)

        if a.shape[0] < self.window:  # if the size of ndarray "a" is less than the window size

            # get 16-bin histogram "c" and bin index "Hbin" from ndarray "a"
            Hbin, c = self._entropy_bin_counts(a)

            # save histogram in output adding counts at the specified bin index
            output[Hbin, :] += c

        else:
            # strided trick from here: http://www.rigtorp.se/2011/01/01/rolling-statistics-numpy.html

            # get shape for stride_tricks.as_strided
            shape = a.shape[:-1] + (a.shape[-1] - self.window + 1, self.window)

            # get strides stride_tricks.as_strided: bytes to step in each dimension when traversing the ndarray
            strides = a.strides + (a.strides[-1],)

            # create a view into the "a" ndarray with the given shape and strides, getting one row every "step" steps
            blocks = np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides)[::self.step, :]

            # from the blocks, compute histogram
            for block in blocks:
                # get 16-bin histogram "c" and bin index "Hbin" from ndarray block
                Hbin, c = self._entropy_bin_counts(block)

                # save histogram in output adding counts at the specified bin index
                output[Hbin, :] += c

        # get a copy of the output ndarray collapsed into one dimension,
        # then convert the numpy ndarray to a python list and return it
        return output.flatten().tolist()

    def process_raw_features(self,
                             raw_obj):  # byte entropy histogram raw features

        # create counts ndarray from the raw byte entropy histogram interpreting the values as floats
        counts = np.array(raw_obj, dtype=np.float32)

        # sum all counts together
        counts_sum = counts.sum()

        # normalize counts dividing them by the total sum
        normalized = counts / counts_sum

        # return normalized histogram
        return normalized

class SectionInfo(FeatureType):
    """ Information about section names, sizes and entropy.
    Uses hashing trick to summarize all this section info into a feature vector.
    """

    name = 'section'
    dim = 5 + 50 + 50 + 50 + 50 + 50

    def __init__(self):
        """ Initialize SectionInfo class. """

        super(FeatureType, self).__init__()

    @staticmethod
    def _properties(s):  # lief binary section
        """ Get section characteristics list.

        Args:
            s: Lief binary section
        Returns:
            Section characteristics list.
        """

        # get section characteristics list, throwing away any string preceding the last "."
        return [str(c).split('.')[-1] for c in s.characteristics_lists]

    def raw_features(self,
                     bytez,  # PE file binary data
                     lief_binary):  # lief parsing of PE file binaries

        # if the leaf parsing is None just return empty section info feature
        if lief_binary is None:
            return {"entry": "", "sections": []}

        # properties of entry point, or if invalid, the first executable section
        try:
            # get entry section name
            entry_section = lief_binary.section_from_offset(lief_binary.entrypoint).name
        except lief.not_found:
            # bad entry point, let's find the first executable section
            entry_section = ""
            for s in lief_binary.sections:  # for all the binary sections
                # if the current section is executable (MEM_EXECUTE is in the section characteristics)
                if lief.PE.SECTION_CHARACTERISTICS.MEM_EXECUTE in s.characteristics_lists:
                    # set entry section name
                    entry_section = s.name
                    break

        # set entry name in raw object dictionary
        raw_obj = {"entry": entry_section, "sections": [{
            'name': s.name,  # section name
            'size': s.size,  # section size
            'entropy': s.entropy,  # section entropy
            'vsize': s.virtual_size,  # section virtual size
            'props': self._properties(s)  # section properties
        } for s in lief_binary.sections]}

        # for every section add its properties to "sections" vector in raw object dictionary

        # return raw object dictionary
        return raw_obj

    def process_raw_features(self,
                             raw_obj):  # section info raw features

        # get sections (vector) from raw object dictionary
        sections = raw_obj['sections']

        # set general info
        general = [
            len(sections),  # total number of sections
            # number of sections with nonzero size
            sum(1 for s in sections if s['size'] == 0),
            # number of sections with an empty name
            sum(1 for s in sections if s['name'] == ""),
            # number of RX sections
            sum(1 for s in sections if 'MEM_READ' in s['props'] and 'MEM_EXECUTE' in s['props']),
            # number of W sections
            sum(1 for s in sections if 'MEM_WRITE' in s['props'])
        ]

        # gross characteristics of each section

        # get sections' names and sizes
        section_sizes = [(s['name'], s['size']) for s in sections]
        # use Feature Hasher to do the hashing trick on section_sizes
        section_sizes_hashed = FeatureHasher(50, input_type="pair").transform([section_sizes]).toarray()[0]
        # get sections' entropies
        section_entropy = [(s['name'], s['entropy']) for s in sections]
        # do the hashing trick on section_entropy
        section_entropy_hashed = FeatureHasher(50, input_type="pair").transform([section_entropy]).toarray()[0]
        # get sections' virtual sizes
        section_vsize = [(s['name'], s['vsize']) for s in sections]
        # do the hashing trick on section_vsize
        section_vsize_hashed = FeatureHasher(50, input_type="pair").transform([section_vsize]).toarray()[0]
        # do the hashing trick on sections' entry names
        entry_name_hashed = FeatureHasher(50, input_type="string").transform([raw_obj['entry']]).toarray()[0]
        # get entry sections' characteristics (properties)
        characteristics = [p for s in sections for p in s['props'] if s['name'] == raw_obj['entry']]
        # do the hashing trick on entry sections' characteristics
        characteristics_hashed = FeatureHasher(50, input_type="string").transform([characteristics]).toarray()[0]

        # concatenate characteristics ndarrays in sequence horizontally,
        # then copy of the array casting its values to float; then return it
        return np.hstack([
            general, section_sizes_hashed, section_entropy_hashed, section_vsize_hashed, entry_name_hashed,
            characteristics_hashed
        ]).astype(np.float32)

class ImportsInfo(FeatureType):
    """ Information about imported libraries and functions from the import address table.
    Note that the total number of imported functions is contained in GeneralFileInfo.
    """

    name = 'imports'
    dim = 1280

    def __init__(self):
        """ Initialize ImportsInfo class. """

        super(FeatureType, self).__init__()

    def raw_features(self,
                     bytez,  # PE file binary data
                     lief_binary):  # lief parsing of PE file binaries

        # initialize imports as an empty dictionary
        imports = {}
        if lief_binary is None:  # if lief binary object is None then return the empty imports raw feature
            return imports

        # for each imported library
        for lib in lief_binary.imports:
            # if the library is not yet in the imports dictionary
            if lib.name not in imports:
                imports[lib.name] = []  # libraries can be duplicated in listing, extend instead of overwrite

            # Clipping assumes there are diminishing returns on the discriminatory power of imported functions
            # beyond the first 10000 characters, and this will help limit the dataset size
            for entry in lib.entries:  # for each imported entry (function) in the library
                if entry.is_ordinal:  # if ordinal is used
                    imports[lib.name].append("ordinal" + str(entry.ordinal))  # append entry ordinal value
                else:
                    imports[lib.name].append(entry.name[:10000])  # append entry name truncated to the first 10000 chars

        # return imports dictionary
        return imports

    def process_raw_features(self,
                             raw_obj):  # imports info raw features

        # get unique libraries
        libraries = list(set([lib.lower() for lib in raw_obj.keys()]))
        # do the hashing trick on libraries names
        libraries_hashed = FeatureHasher(256, input_type="string").transform([libraries]).toarray()[0]

        # generate a string like "kernel32.dll:CreateFileMappingA" for each imported function
        imports = [lib.lower() + ':' + e for lib, elist in raw_obj.items() for e in elist]
        # do the hashing trick on imports
        imports_hashed = FeatureHasher(1024, input_type="string").transform([imports]).toarray()[0]

        # return two separate elements: libraries (alone) and fully-qualified names of imported functions
        # stacked together in a single, one dimensional, array with values of type float
        return np.hstack([libraries_hashed, imports_hashed]).astype(np.float32)

class ExportsInfo(FeatureType):
    """ Information about exported functions.
    Note that the total number of exported functions is contained in GeneralFileInfo.
    """

    name = 'exports'
    dim = 128

    def __init__(self):
        """ Initialize ExportsInfo class. """

        super(FeatureType, self).__init__()

    def raw_features(self,
                     bytez,  # PE file binary data
                     lief_binary):  # lief parsing of PE file binaries

        # if lief binary object is None return an empty vector
        if lief_binary is None:
            return []

        # Clipping assumes there are diminishing returns on the discriminatory power of exports beyond
        # the first 10000 characters, and this will help limit the dataset size
        if LIEF_EXPORT_OBJECT:
            # export is an object with .name attribute (0.10.0 and later) -> get export functions' names (clipped)
            clipped_exports = [export.name[:10000] for export in lief_binary.exported_functions]
        else:
            # export is a string (LIEF 0.9.0 and earlier)
            clipped_exports = [export[:10000] for export in lief_binary.exported_functions]

        # return exports raw feature
        return clipped_exports

    def process_raw_features(self,
                             raw_obj):  # exports info raw features

        # do the hashing trick on exported functions raw feature
        exports_hashed = FeatureHasher(128, input_type="string").transform([raw_obj]).toarray()[0]

        # return exported functions feature vector
        return exports_hashed.astype(np.float32)

class GeneralFileInfo(FeatureType):
    """ General information about the file. """

    name = 'general'
    dim = 10

    def __init__(self):
        """ Initialize GeneralFileInfo class. """

        super(FeatureType, self).__init__()

    def raw_features(self,
                     bytez,  # PE file binary data
                     lief_binary):  # lief parsing of PE file binaries

        # if lief binary object is None return a general file info dictionary with default (zero) values
        if lief_binary is None:
            return {
                'size': len(bytez),  # the only property we can compute
                'vsize': 0,
                'has_debug': 0,
                'exports': 0,
                'imports': 0,
                'has_relocations': 0,
                'has_resources': 0,
                'has_signature': 0,
                'has_tls': 0,
                'symbols': 0
            }

        # return general file info dictionary
        return {
            'size': len(bytez),  # get PE file binaries length
            'vsize': lief_binary.virtual_size,  # get virtual size
            'has_debug': int(lief_binary.has_debug),  # get whether the current binary has a Debug object
            'exports': len(lief_binary.exported_functions),  # get number of exported functions
            'imports': len(lief_binary.imported_functions),  # get number of imported functions
            'has_relocations': int(lief_binary.has_relocations),  # get whether the current binary uses Relocation
            'has_resources': int(lief_binary.has_resources),  # get whether the current binary has a Resources object
            # get whether the binary has signatures
            'has_signature': int(lief_binary.has_signatures) if LIEF_HAS_SIGNATURE else int(lief_binary.has_signature),
            'has_tls': int(lief_binary.has_tls),  # get whether the current binary has a TLS object
            'symbols': len(lief_binary.symbols),  # get number of binary's symbols
        }

    def process_raw_features(self,
                             raw_obj):  # general file info raw features

        # return one single ndarray of float values, concatenating together the raw general file info features
        return np.asarray([
            raw_obj['size'], raw_obj['vsize'], raw_obj['has_debug'], raw_obj['exports'], raw_obj['imports'],
            raw_obj['has_relocations'], raw_obj['has_resources'], raw_obj['has_signature'], raw_obj['has_tls'],
            raw_obj['symbols']
        ], dtype=np.float32)

class HeaderFileInfo(FeatureType):
    """ Machine, architecture, OS, linker and other information extracted from header. """

    name = 'header'
    dim = 62

    def __init__(self):
        """ Initialize HeaderFileInfo class. """

        super(FeatureType, self).__init__()

    def raw_features(self,
                     bytez,  # PE file binary data
                     lief_binary):  # lief parsing of PE file binaries

        # instantiate header file info raw object dictionary
        raw_obj = {'coff': {'timestamp': 0, 'machine': "", 'characteristics': []},
                   'optional': {
                       'subsystem': "",
                       'dll_characteristics': [],
                       'magic': "",
                       'major_image_version': 0,
                       'minor_image_version': 0,
                       'major_linker_version': 0,
                       'minor_linker_version': 0,
                       'major_operating_system_version': 0,
                       'minor_operating_system_version': 0,
                       'major_subsystem_version': 0,
                       'minor_subsystem_version': 0,
                       'sizeof_code': 0,
                       'sizeof_headers': 0,
                       'sizeof_heap_commit': 0
                   }}

        # initialize header file info raw object dictionary with default values

        # if lief binary object is None return default raw object
        if lief_binary is None:
            return raw_obj

        # get coff header timestamp (when the file was created)
        raw_obj['coff']['timestamp'] = lief_binary.header.time_date_stamps
        # get the coff header machine (throwing away any string before the last ".")
        raw_obj['coff']['machine'] = str(lief_binary.header.machine).split('.')[-1]
        # get coff header characteristics (throwing away any string before the last ".")
        raw_obj['coff']['characteristics'] = [str(c).split('.')[-1] for c in lief_binary.header.characteristics_list]
        # get the optional header subsystem required to run this image (throwing away any string before the last ".")
        raw_obj['optional']['subsystem'] = str(lief_binary.optional_header.subsystem).split('.')[-1]
        # get the optional header dll characteristics (throwing away any string before the last ".")
        raw_obj['optional']['dll_characteristics'] = [
            str(c).split('.')[-1] for c in lief_binary.optional_header.dll_characteristics_lists
        ]
        # get the optional header magic value (throwing away any string before the last ".")
        raw_obj['optional']['magic'] = str(lief_binary.optional_header.magic).split('.')[-1]
        # get the optional header major image version
        raw_obj['optional']['major_image_version'] = lief_binary.optional_header.major_image_version
        # get the optional header minor image version
        raw_obj['optional']['minor_image_version'] = lief_binary.optional_header.minor_image_version
        # get the optional header major linker version
        raw_obj['optional']['major_linker_version'] = lief_binary.optional_header.major_linker_version
        # get the optional header minor linker version
        raw_obj['optional']['minor_linker_version'] = lief_binary.optional_header.minor_linker_version
        # get the optional header major operating system version required
        raw_obj['optional'][
            'major_operating_system_version'] = lief_binary.optional_header.major_operating_system_version
        # get the optional header minor operating system version required
        raw_obj['optional'][
            'minor_operating_system_version'] = lief_binary.optional_header.minor_operating_system_version
        # get the optional header majosr subsystem version
        raw_obj['optional']['major_subsystem_version'] = lief_binary.optional_header.major_subsystem_version
        # get the optional header mino subsystem version
        raw_obj['optional']['minor_subsystem_version'] = lief_binary.optional_header.minor_subsystem_version
        # get the optional header size of the code (text) section (or sum of code sizes if there are multiple sections)
        raw_obj['optional']['sizeof_code'] = lief_binary.optional_header.sizeof_code
        # get the optional header combined size of an MS-DOS stub, PE header, and section headers
        raw_obj['optional']['sizeof_headers'] = lief_binary.optional_header.sizeof_headers
        # get the optional header size of the local heap space to commit
        raw_obj['optional']['sizeof_heap_commit'] = lief_binary.optional_header.sizeof_heap_commit

        # return raw object dictionary
        return raw_obj

    def process_raw_features(self,
                             raw_obj):  # header file info raw features

        # return one single 1-D ndarray of float values obtained concatenating along one dimension the raw features,
        # some of which are transformed through the hashing trick
        return np.hstack([
            raw_obj['coff']['timestamp'],
            FeatureHasher(10, input_type="string").transform([[raw_obj['coff']['machine']]]).toarray()[0],
            FeatureHasher(10, input_type="string").transform([raw_obj['coff']['characteristics']]).toarray()[0],
            FeatureHasher(10, input_type="string").transform([[raw_obj['optional']['subsystem']]]).toarray()[0],
            FeatureHasher(10, input_type="string").transform([raw_obj['optional']['dll_characteristics']]).toarray()[0],
            FeatureHasher(10, input_type="string").transform([[raw_obj['optional']['magic']]]).toarray()[0],
            raw_obj['optional']['major_image_version'],
            raw_obj['optional']['minor_image_version'],
            raw_obj['optional']['major_linker_version'],
            raw_obj['optional']['minor_linker_version'],
            raw_obj['optional']['major_operating_system_version'],
            raw_obj['optional']['minor_operating_system_version'],
            raw_obj['optional']['major_subsystem_version'],
            raw_obj['optional']['minor_subsystem_version'],
            raw_obj['optional']['sizeof_code'],
            raw_obj['optional']['sizeof_headers'],
            raw_obj['optional']['sizeof_heap_commit'],
        ]).astype(np.float32)

class StringExtractor(FeatureType):
    """ Extracts strings from raw byte stream. """

    name = 'strings'
    dim = 1 + 1 + 1 + 96 + 1 + 1 + 1 + 1 + 1

    def __init__(self):
        """ Initialize StringExtractor class. """

        super(FeatureType, self).__init__()
        # compile a bunch of regular expression patterns into regular expression objects to be later used for matching

        # all consecutive runs of 0x20 - 0x7f that are 5+ characters
        self._allstrings = re.compile(b'[\x20-\x7f]{5,}')
        # occurances of the string 'C:\'.  Not actually extracting the path
        self._paths = re.compile(b'c:\\\\', re.IGNORECASE)
        # occurances of http:// or https://.  Not actually extracting the URLs
        self._urls = re.compile(b'https?://', re.IGNORECASE)
        # occurances of the string prefix HKEY_.  No actually extracting registry names
        self._registry = re.compile(b'HKEY_')
        # crude evidence of an MZ header (dropper?) somewhere in the byte stream
        self._mz = re.compile(b'MZ')

    def raw_features(self,
                     bytez,  # PE file binary data
                     lief_binary):  # lief parsing of PE file binaries

        # find all occurrencies of _allstrings in the PE file binary data
        allstrings = self._allstrings.findall(bytez)
        if allstrings:  # if at least one string has been matched
            # statistics about strings:

            # get the length of each string
            string_lengths = [len(s) for s in allstrings]
            # compute average string length
            avlength = sum(string_lengths) / len(string_lengths)
            # map printable characters 0x20 - 0x7f to an int array consisting of 0-95, inclusive
            as_shifted_string = [b - ord(b'\x20') for b in b''.join(allstrings)]
            # compute histogram count for the 96 letters (distribution of characters in printable strings)
            c = np.bincount(as_shifted_string, minlength=96)
            # sum counts
            csum = c.sum()
            # compute character probabilities (as floats)
            p = c.astype(np.float32) / csum
            # get non-zero bin indexes
            wh = np.where(c)[0]
            # calculate entropy
            H = np.sum(-p[wh] * np.log2(p[wh]))
        else:  # if no strings were matched set to zero (default value) some variables
            avlength = 0
            c = np.zeros((96,), dtype=np.float32)  # histogram of all zeros
            H = 0
            csum = 0

        # return raw object dictionary with the computed information
        return {
            'numstrings': len(allstrings),  # total number of strings found in the PE file
            'avlength': avlength,  # average string length
            'printabledist': c.tolist(),  # non-normalized printable characters frequency histogram
            'printables': int(csum),  # total number of printable characters found
            'entropy': float(H),  # entropy
            'paths': len(self._paths.findall(bytez)),  # number of found paths
            'urls': len(self._urls.findall(bytez)),  # number of found urls
            'registry': len(self._registry.findall(bytez)),  # number of found registry names
            'MZ': len(self._mz.findall(bytez))  # number of found MZ headers
        }

    def process_raw_features(self,
                             raw_obj):  # string extractor raw features

        # set histogram divisor as csum (printables) if it was > 0, to 1.0 otherwise
        hist_divisor = float(raw_obj['printables']) if raw_obj['printables'] > 0 else 1.0

        # return one single 1-D ndarray of float values obtained concatenating together the raw features
        # (normalizing the printable characters histogram)
        return np.hstack([
            raw_obj['numstrings'],
            raw_obj['avlength'],
            raw_obj['printables'],
            np.asarray(raw_obj['printabledist']) / hist_divisor,  # divide distribution by the previously set divisor
            raw_obj['entropy'],
            raw_obj['paths'],
            raw_obj['urls'],
            raw_obj['registry'],
            raw_obj['MZ']
        ]).astype(np.float32)

class DataDirectories(FeatureType):
    """ Extracts size and virtual address of the first 15 data directories. """

    name = 'datadirectories'
    dim = 15 * 2

    def __init__(self):
        """ Initialize DataDirectories class. """

        super(FeatureType, self).__init__()

        # define data directory names order
        self._name_order = [
            "EXPORT_TABLE", "IMPORT_TABLE", "RESOURCE_TABLE", "EXCEPTION_TABLE", "CERTIFICATE_TABLE",
            "BASE_RELOCATION_TABLE", "DEBUG", "ARCHITECTURE", "GLOBAL_PTR", "TLS_TABLE", "LOAD_CONFIG_TABLE",
            "BOUND_IMPORT", "IAT", "DELAY_IMPORT_DESCRIPTOR", "CLR_RUNTIME_HEADER"
        ]

    def raw_features(self,
                     bytez,  # PE file binary data
                     lief_binary):  # lief parsing of PE file binaries

        # instantiate output vector
        output = []

        # if lief binary object is None return empty output raw object
        if lief_binary is None:
            return output

        # for each data directory
        for data_directory in lief_binary.data_directories:
            # append info to output vector
            output.append({
                # set data directory type name (removing the "DATA_DIRECTORY." prefix)
                "name": str(data_directory.type).replace("DATA_DIRECTORY.", ""),
                "size": data_directory.size,  # set data directory size
                "virtual_address": data_directory.rva  # set data directory virtual address
            })

        # return output raw feature vector
        return output

    def process_raw_features(self,
                             raw_obj):  # data dictionaries raw features

        # initialize features vector with size equal to 2 times the number of data directory types
        # with all zeros (as floats)
        features = np.zeros(2 * len(self._name_order), dtype=np.float32)

        # iterate for a number of times equal to the number of data directory types
        for i in range(len(self._name_order)):
            if i < len(raw_obj):
                # set data directory size to the feature vector at an even position
                features[2 * i] = raw_obj[i]["size"]
                # set data directory virtual address to the feature vector at an odd position
                features[2 * i + 1] = raw_obj[i]["virtual_address"]

        # return data directory feature vector
        return features

class PEFeatureExtractor(object):
    """ Extract useful features from a PE file, and return as a vector of fixed size. """

    def __init__(self,
                 feature_version=2,  # EMBER feature version
                 print_feature_warning=True):  # whether to print warnings or not
        """ Initialize PEFeatureExtractor class.

        Args:
            feature_version: EMBER feature version
            print_feature_warning: Whether to print warnings or not
        """

        # define features to extract from PE file binaries
        self.features = [
            ByteHistogram(),
            ByteEntropyHistogram(),
            StringExtractor(),
            GeneralFileInfo(),
            HeaderFileInfo(),
            SectionInfo(),
            ImportsInfo(),
            ExportsInfo()
        ]

        # check EMBER feature version selected against current lief version
        if feature_version == 1:
            if not lief.__version__.startswith("0.8.3"):
                if print_feature_warning:
                    print("WARNING: EMBER feature version 1 were computed using lief version 0.8.3-18d5b75")
                    print("WARNING: lief version {} found instead. There may be slight inconsistencies"
                          .format(lief.__version__))
                    print("WARNING: in the feature calculations.")
        elif feature_version == 2:
            self.features.append(DataDirectories())  # append another type of feature specific to version 2
            if not lief.__version__.startswith("0.9.0"):
                if print_feature_warning:
                    print("WARNING: EMBER feature version 2 were computed using lief version 0.9.0-")
                    print("WARNING: lief version {} found instead. There may be slight inconsistencies"
                          .format(lief.__version__))
                    print("WARNING: in the feature calculations.")
        else:
            raise Exception(f"EMBER feature version must be 1 or 2. Not {feature_version}")

        # compute features total dimension
        self.dim = sum([fe.dim for fe in self.features])

    def raw_features(self,
                     bytez):  # PE file binary data

        # define all lief errors we want to intercept
        lief_errors = (lief.bad_format, lief.bad_file, lief.pe_error, lief.parser_error, lief.read_out_of_bound,
                       RuntimeError)
        try:
            lief_binary = lief.PE.parse(list(bytez))  # Parse the given PE file binaries and return a Binary object
        except lief_errors as e:  # if any of the previously defined lief errors is raised
            logger.warn("lief error: {}. Skipping file.".format(str(e)))  # print the error
            return None
        except Exception:  # if any other exception (KeyboardInterrupt, SystemExit, ValueError) is raised:
            raise  # raise exception

        # calculate sha256 hash (hex) digest of the PE file binaries
        features = {"sha256": hashlib.sha256(bytez).hexdigest()}

        # compute all the other raw features and append them to the features dictionary
        features.update({fe.name: fe.raw_features(bytez, lief_binary) for fe in self.features})

        # return computed raw features for the current PE file binaries
        return features

    def process_raw_features(self,
                             raw_obj):  # dictionary of raw features

        # compute feature vector by processing the raw features in the raw features dictionary
        feature_vectors = [fe.process_raw_features(raw_obj[fe.name]) for fe in self.features]

        # concatenate feature vectors (horizontally) in a single numpy array,
        # and return a copy of the array casting its values to float32
        return np.hstack(feature_vectors).astype(np.float32)

    def feature_vector(self,
                       bytez):  # PE file data

        # get raw features from PE file data,
        # then generate feature vector from raw features
        return self.process_raw_features(self.raw_features(bytez))


#Hacer un programa que procese un total de 100 personas y establecer cuantas son mujeres
# mayores de edad y cuantos hombres menores de edad. Deberá sacar el hombre y la mujer
# con menor edad, el hombre y la mujer con mayor edad, promedio de edades de las mujeres
# y promedio de edades de los hombres.
from random import randint
'''
#generamos un set aleatorio de cien personas (hombres(1) y mujeres (0)) con edades aleatorias
#entre 0 y 100 años
listamujeres = []
listahombres = []

# 100 personas
for i in range(1,101):
    # genero
    genero = randint(0,1)
    # edad
    edad = randint(0,100)
    # creamos una lista de mujeres y otra de hombres
    if genero == 0:
        listamujeres.append(edad)
    else:
        listahombres.append(edad)

#print(listamujeres, listahombres)
'''
#pasamos lo anterior a listas de comprension
# mujeres y hombres aleatorios en 100 personas
genero = [randint(0,1) for i in range(1,101)]
#lista de edades entre 0 y 100 anos mujeres
listamujeres = [randint(0,100) for g in genero if g==0]
#lista de edades entre 0 y 100 anos hombres
listahombres = [randint(0,100) for g in genero if g==1]

# porcentaje por generos
print(f'Mujeres: {len(listamujeres)}% y hombres:{len(listahombres)}% ')

#mujeres mayores de edad
'''
mujeresmayores = 0
for edad in listamujeres:
    if edad >= 18:
        mujeresmayores += 1
'''
mujeresmayores = [edad for edad in listamujeres if edad >= 18]

print(f"Hay {len(mujeresmayores)} mujeres mayores de edad")

#hombres menores de edad
'''
hombresmenores = 0
for edad in listahombres:
    if edad < 18:
        hombresmenores += 1
'''
hombresmenores = [edad for edad in listahombres if edad < 18]
print(f"Hay {len(hombresmenores)} hombres menores de edad")

#hombre y mujer de mayor y menor edad
def mayormenor(lista,genero):
    mayoredad = max(lista)
    menoredad = min(lista)
    print(f'{genero} de mayor edad tiene {mayoredad} años y de menor edad, {menoredad} años')

mayormenor(listamujeres, 'Mujer')
mayormenor(listahombres, 'Hombre')

# promedio edad mujeres y promedio edad hombres
def promedioedad(lista, genero):
    '''
    contadoredad = 0
    for edad in lista:
        contadoredad += edad
    '''
    contadoredad = sum([edad for edad in lista])
    promedio = contadoredad / len(listamujeres)
    print("Edad promedio en {g}: {p:1.2f} años".format(g= genero,p=promedio))

promedioedad(listamujeres,'mujeres')
promedioedad(listahombres,'hombres')


# https://www.hackerrank.com/challenges/mini-max-sum
# Given array, find the min and max values that can be calculated by summing exactly four of the five integers
def mini_max_sum(arr):
    aux = sum(arr)
    print(aux - max(arr), aux - min(arr))

if __name__ == '__main__':
    ar = list(map(int, input().rstrip().split()))

    mini_max_sum(ar)


from math import ceil
from time import sleep
from serial import Serial
import RPi.GPIO as gpio			# GPIO control module
from support.ax12 import Ax12
from numpy import pi

class Servo:

	# /////////////////////////////// angle limits (folding fan + 1-2-3 joints)
	LIM_FF = [60,126]
	LIM_1J = [50,150]
	LIM_2J = [50,300]
	LIM_3J = [50,300]

	# /////////////////////////////// offset values (folding fan + 1-2-3 joints)
	OS_FF = 60
	OS_1J = 114
	OS_2J = 150
	OS_3J = 163

	# /////////////////////////////// conversion costants
	EXA_TO_DEG = 300/1023
	DEG_TO_EXA = 1023/300
	RAD_TO_DEG = 180/pi
	DEG_TO_RAD = pi/180

	# /////////////////////////////// other constants
	DELAY = 0.1
	MIN_ANGLE = 0
	MAX_ANGLE = 300
	MAX_LOOP = 20
	TOT_SERVO = 31
	TOT_LEGS = 6

	# Custom error class to report errors on checkServo
	class servoError(Exception):
		pass

	def __init__(self,used_servo=3,minID=0,maxID=20,verbose=False):
		self.min_angle = Servo.MIN_ANGLE
		self.max_angle = Servo.MAX_ANGLE
		self.tot_servo = Servo.TOT_SERVO
		self.tot_legs = Servo.TOT_LEGS
		self.ax = Ax12()
		try:
			id_list,curr_legs = self.checkServo(used_servo,minID,maxID,verbose) # create ID list
			self.id = id_list
			self.num = len(id_list)
			self.curr_legs = curr_legs
			self.lim_cw = [self.min_angle] * self.tot_servo
			self.lim_ccw = [self.max_angle] * self.tot_servo
			self.offset = [self.min_angle] * self.tot_servo
			self.curr_angles = [self.min_angle] * self.num
			self.setup()
		except Exception as detail:
			print(str(detail))
			pass

	def setup(self):
		self.setOffset()
		self.updateAngles() # update self.curr_angle
		print("Servo ID list = ", self.id)
		print("Starting angles = ", self.curr_angles)
		self.setupAngleLim() # setAngleLim on all servos
		self.printTorqueLimit()
		for i in range(self.num):
			self.ax.setVoltageLimit(self.id[i],88,130)
			self.ax.setLedAlarm(self.id[i],"Input Voltage")
			self.ax.setTorqueLimit(self.id[i],1023) # set maximum torque for all servos
			v = self.ax.readVoltageLowerLimit(self.id[i])
			print(f"Input Voltage Lower Limit servo ID {self.id[i]} = ({v})", end="     ")
		print()
		self.printAngles()

	def checkServo(self,used_servo,minID=0,maxID=253,verbose=False):
		id_list = self.ax.learnServos(minID,maxID,verbose)
		confirm = len(id_list) == used_servo
		curr_legs = 0
		try:
			assert confirm == 1
		except:
			e = "\n================================================================================\n"
			e += "Error: discrepancy between given variable used_servo and number of servos found.\n"
			e += "Variable used_servo: " + str(used_servo) + "\n"
			e += "Actual number of servos: " + str(len(id_list)) + "\n"
			e += "Actual list of servos: " + str(id_list) + "\n"
			e += "================================================================================\n"
			raise Servo.servoError(e)
		if id_list[0] == 0: # the first is the folding fan servo
			curr_legs = ceil((used_servo-1)/3) # -1 to single out the folding fan servo from the computation
		else:
			curr_legs = ceil(used_servo/3)
		return id_list,curr_legs

	def updateAngles(self):
		curr_angles = self.curr_angles
		for i in range(self.num):
			r = self.ax.readPosition(self.id[i])
			count = 0
			while r == -1 and count < Servo.MAX_LOOP:
				r = self.ax.readPosition(self.id[i]) # try again if servo didn't respond
				count += 1							 # to avoid infinite loop
			if count >= Servo.MAX_LOOP:
				print("Function readPosition inside updateAngles failed " + str(Servo.MAX_LOOP) + " times for servo " + str(self.id[i]))
				# self.ax.ping(self.id[i]) # checks again for timeout error (ping function bypasses -1 error return)
				# r = self.ax.readPosition(self.id[i])
			else:
				r_deg = int(r * Servo.EXA_TO_DEG)
				if r_deg >= 0 and r_deg <= 300:
					curr_angles[i] = r_deg
		self.curr_angles = curr_angles

	def checkVoltage(self,i):
		r = self.ax.readVoltage(self.id[i])
		count = 0
		while r == -1 and count < Servo.MAX_LOOP:
			r = self.ax.readVoltage(self.id[i])	# try again if servo didn't respond
			count += 1							# to avoid infinite loop
		voltage = r/10
		return voltage

	def setAngleLim(self,i,lim_vett):
		lim_cw,lim_ccw = lim_vett	# clockwise - counter-clockwise
		self.lim_cw[i], self.lim_ccw[i] = lim_cw, lim_ccw
		lim_cw,lim_ccw = self.rangeAngleEx(lim_cw),self.rangeAngleEx(lim_ccw)
		self.ax.setAngleLimit(self.id[i], lim_cw, lim_ccw)
		sleep(Servo.DELAY)

	def setupAngleLim(self):
		if self.id[0] == 0:
			self.setAngleLim(0, Servo.LIM_FF) # servo 0 folding fan
		for i in range(1,self.curr_legs+1):
			self.setAngleLim(i*3-2, Servo.LIM_1J)
			if self.num > 1:
				self.setAngleLim(i*3-1, Servo.LIM_2J)
				self.setAngleLim(i*3, Servo.LIM_3J)
		self.printAngleLim()

	def setOffset(self):
		os = [0] * self.tot_servo
		os[0] = Servo.OS_FF	# servo 0 folding fan
		for i in range(1,self.tot_legs+1):
			os[i*3-2] = Servo.OS_1J
			os[i*3-1] = Servo.OS_2J
			os[i*3] =	Servo.OS_3J
		self.offset = os

	def conv_from_servo(self,i,a):
		return (a - self.offset[i]) * Servo.DEG_TO_RAD

	def conv_to_servo(self,i,a):
		return a * Servo.RAD_TO_DEG + self.offset[i]

	def rangeAngleEx(self,a):
		if a < Servo.MIN_ANGLE: a = Servo.MIN_ANGLE
		if a > Servo.MAX_ANGLE: a = Servo.MAX_ANGLE
		a = int(a * Servo.DEG_TO_EXA)
		return a

	def printAngleLim(self):
		for i in range(self.num):
			if self.id[0] == 0:
				j = i
			else:
				j = i+1
			print(f"Angle limits servo ID {self.id[i]} = ({self.lim_cw[j]}, {self.lim_ccw[j]})")

	def checkChangedAngle(self,i,a):
		ca = self.curr_angles[i]
		if ca > a + 1 or ca < a - 1:
			return 1
		else:
			return 0

	def checkChangedAngles(self,a):
		for i in range(self.num):
			ca = self.curr_angles[i]
			if ca > a[i] + 1 or ca < a[i] - 1:
				return 1
		return 0

	def printAngles(self):
		for i in range(self.num):
			a = self.curr_angles[i]
			print(f"Angle servo ID {self.id[i]} = ({a})", end="")
			if a < 10:				print("       ", end="")
			if a >= 10 and a < 100:	print("      ", end="")
			else:					print("     ", end="")
		print("\n")

	def printVoltage(self):
		for i in range(self.num):
			v = self.checkVoltage(i)
			print(f"Voltage servo ID {self.id[i]} = ({v})", end="     ")
		print("\n")

	def setAngle(self,i,a,speed=100): # i is always the position in order, not the ID of the servo
		a = self.rangeAngleEx(a)
		self.ax.moveSpeed(self.id[i], a, speed)
		self.updateAngles()

	def setAngleAll(self,b,speed=50):
		for i in range(self.num):
			a = self.rangeAngleEx(b[i])
			self.ax.moveSpeed(self.id[i], a, speed)
		self.updateAngles()

	def enableTorqueAll(self):
		for i in range(self.num):
			self.ax.setTorqueStatus(self.id[i],1)

	def disableTorqueAll(self):
		for i in range(self.num):
			self.ax.setTorqueStatus(self.id[i],0)

	def printTorqueStatus(self):
		for i in range(self.num):
			print(f"Torque enable servo ID {self.id[i]} = ({self.ax.readTorqueStatus(self.id[i])})", end="     ")
		print()

	def printTorqueLimit(self):
		for i in range(self.num):
			print(f"Torque limit servo ID {self.id[i]} = ({self.ax.readTorqueLimit(self.id[i])})", end="     ")
		print()

	def changeID(self,i,newID):
		self.ax.setID(self.id[i], newID)
		self.id[i] = newID
		sleep(Servo.DELAY)
		print("Updated list of ID = ", self.id)

	def blink(self,time=1):
		for i in self.id: self.ax.setLedStatus(i,1)
		sleep(time)
		for i in self.id: self.ax.setLedStatus(i,0)
		sleep(time)

	def offLED(self):
		for i in self.id:
			self.ax.setLedStatus(i, 0)
			sleep(Servo.DELAY)


# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right
class Solution:
    def evaluateTree(self, root: Optional[TreeNode]) -> bool:
        def help(root):
            if root.val==1 or root.val==0:
                return root.val
            if root.val==2:
                l=help(root.left)
                r= help(root.right)
                if l==1 or r==1:
                    root.val= 1
                else:
                    root.val=0
            if root.val==3:
                l=help(root.left)
                r= help(root.right)
                if l==1 and r==1:
                    root.val= 1
                else:
                    root.val=0
            return root.val
        return help(root)


#!/usr/bin/env python
from setuptools import find_packages, setup

install_requires = [
    'wagtail>=2.0',
]

docs_require = [
    'sphinx>=1.4.0',
]

tests_require = [
    'pytest-cov==2.4.0',
    'pytest-django==3.1.2',
    'pytest==3.0.5',
    'requests-mock==1.1.0',

    # Linting
    'isort==4.2.5',
    'flake8==3.0.3',
    'flake8-blind-except==0.1.1',
    'flake8-debugger==1.4.0',
    'flake8-imports==0.1.0',
]

setup(
    name='wagtail-audit-trail',
    version='1.3.0',
    description="Wagtail audit trail",
    long_description=open('README.rst', 'r').read(),
    url='https://github.com/labd/wagtail-audit-trail',
    author="Lab Digital",
    author_email="opensource@labdigital.nl",
    install_requires=install_requires,
    tests_require=tests_require,
    extras_require={
        'docs': docs_require,
        'test': tests_require,
    },
    entry_points={},
    package_dir={'': 'src'},
    packages=find_packages('src'),
    include_package_data=True,
    license='MIT',
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Environment :: Web Environment',
        'Framework :: Django',
        'Framework :: Django :: 1.11',
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.6',
    ],
    zip_safe=False,
)


from collections import defaultdict

class Solution:
    def numberOfBoomerangs(self, points):
        n = len(points)
        dist = defaultdict(lambda: 0)
        for i in range(n):
            x1, y1 = points[i]
            for j in range(i):
                x2, y2 = points[j]
                d = (x1 - x2) ** 2 + (y1 - y2) ** 2
                dist[(i, d)] += 1
                dist[(j, d)] += 1
        ans = 0
        for _, n in dist.items():
            if n > 1:
                ans += n * (n - 1)
        return ans


import art
import random
from game_data import data
from replit import clear

#FUNCIONES:
#funcion para elegir un pj random de la lista:
def random_character(data):
  character = {}
  random_index = random.randint(0, len(data)-1)
  character= data[random_index]
  data.pop(random_index)
  return character

#funcion para comparar valores y retornar el resultado (gana o pierde):
def compare_values(a_char, b_char, user_input):
  user_win = False
  a_option = 'a'
  b_option = 'b'
  if a_char['follower_count'] > b_char['follower_count']:
    if user_input == a_option:
      user_win = True
  elif a_char['follower_count'] < b_char['follower_count']:
    if user_input == b_option:
      user_win = True
  return user_win

#variables fuera del ciclo
a_char = random_character(data)
b_char = random_character(data)
score = 0
wrong_message = f"Sorry, that's wrong. Final score: {score}."
right_message = f"You're right! Current score: {score}."
empty_list = "We ran out of characters! Thank you for playing."
playing = True

#Ciclo
while playing:
  a_compare = f"Compare A: {a_char['name']} a {a_char['description']} from {a_char['country']}.\n"
  b_compare = f"Against B: {b_char['name']} a {b_char['description']} from {b_char['country']}.\n"
  print(art.logo)
  #pantalla con prints
  if score != 0:
    print(f"You're right! Current score: {score}.")
  print(a_compare)
  print(art.vs)
  print(b_compare)
  #eleccion del usuario
  user_input = input("Who has more followers? Type 'A' or 'B': ").lower()
  #se comparan los followers y devuelve el resultado
  result = compare_values(a_char, b_char, user_input)
  #resultado
  if result == True:
    score += 1
    if data == []:
      print(empty_list)
      playing = False
    clear()
    a_char = b_char
    b_char = random_character(data)
  else:
    print(f"Sorry, that's wrong. Final score: {score}.")
    playing = False


from spetlrtools.testing import DataframeTestCase

from spetlr import Configurator
from spetlr.delta import DeltaHandle
from spetlr.etl import Orchestrator
from spetlr.etl.extractors import SimpleExtractor
from spetlr.etl.loaders import SimpleLoader
from spetlr.etl.transformers.SimpleSqlTransformer import SimpleSqlTransformer
from spetlr.spark import Spark
from tests.cluster.transformations import sql

class SimpleSqlTransformerTest(DataframeTestCase):
    @classmethod
    def setUpClass(cls):
        c = Configurator()
        c.clear_all_configurations()
        c.set_debug()
        c.register("db", {"name": "SimpleSqlTransformerTestDb{ID}"})
        c.register("tbl1", {"name": "{db}.tbl1"})
        c.register("tbl2", {"name": "{db}.tbl2"})
        c.register("tbl3", {"name": "{db}.tbl3"})

        db = c.get("db", "name")
        tbl1 = c.get("tbl1", "name")
        tbl2 = c.get("tbl2", "name")
        tbl3 = c.get("tbl3", "name")

        spark = Spark.get()
        spark.sql(f"CREATE DATABASE {db};")
        spark.sql(f"""CREATE TABLE {tbl1} (a int, b int) USING DELTA;""")
        spark.sql(f"""CREATE TABLE {tbl2} (a int, c int) USING DELTA;""")
        spark.sql(f"""CREATE TABLE {tbl3} (a int, b int, c int) USING DELTA;""")

        spark.createDataFrame([(1, 2)], "a int, b int").write.mode(
            "overwrite"
        ).saveAsTable(tbl1)
        spark.createDataFrame([(1, 3)], "a int, c int").write.mode(
            "overwrite"
        ).saveAsTable(tbl2)

    @classmethod
    def tearDownClass(cls) -> None:
        Spark.get().sql(f"""DROP DATABASE {Configurator().get('db','name')} CASCADE""")

    def test_all(self):
        o = Orchestrator()
        o.extract_from(
            SimpleExtractor(DeltaHandle.from_tc("tbl1"), dataset_key="FirstTable")
        )
        o.extract_from(
            SimpleExtractor(DeltaHandle.from_tc("tbl2"), dataset_key="SecondTable")
        )
        o.transform_with(
            SimpleSqlTransformer(
                sql_modue=sql,
                sql_file_pattern="transform",
                dataset_input_keys=["FirstTable", "SecondTable"],
                dataset_output_key="SimpleSqlTransformerTestResult",
            )
        )
        o.load_into(SimpleLoader(DeltaHandle.from_tc("tbl3")))
        o.execute()

        self.assertDataframeMatches(
            DeltaHandle.from_tc("tbl3").read(), None, [(1, 2, 3)]
        )


from queue import Queue
def verticalOrder(self, root):
        topnode = {}

        q = Queue()

        q.put((root, 0))

        while not q.empty():

            node, hzd = q.get()

            try:
                topnode[hzd].append(node.data)
            except:
                topnode[hzd] = [node.data]

            if node.left:

                q.put((node.left, hzd - 1))

            if node.right:

                q.put((node.right, hzd + 1))

        ans = []

        for i in sorted(topnode.keys()):
            for j in topnode[i]:
                ans.append(j)

        return ans


"""
It will be responsible for downloading image  captchas from our website
and save it into our disk

it will store the raw captcha images to our disk

headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}

"""

import requests
#the requests library will make http connections easy and heavily used in python
import argparse
import os
import time

url="https://www.e-zpassny.com/vector/jcaptcha.do"

ap=argparse.ArgumentParser()
ap.add_argument("-u","--url",help="if the images are to be downloaded from url give url",required=False,default=url)
ap.add_argument("-o","--output",required=True,help=" path to save the images")
ap.add_argument("-n","--number",type=int,default=500,help="# of images to download")
args=vars(ap.parse_args())

total=0

for i in range(0,args["number"]):
    try:
        #requesting web page from our script
        r=requests.get(args['url'],timeout=60, headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3'})

        #save the image
        p = os.path.sep.join([args["output"],"{}.jpg".format(str(total).zfill(5))])
        #Str().zfill() Returns a copy of the string with '0' characters padded to the leftside of the given string.
        #ex zfill(3) ==> 000name
        f= open(p,"wb")
        f.write(r.content)
        f.close()

        print("[INFO..] Downloaded image {}".format(total))
        total+=1

    except:
        print("Error downloading image")

    time.sleep(0.2)


#!/usr/bin/env python
# coding: utf-8

# In[27]:

import matplotlib.pyplot as plt

# In[71]:

# line plot
plt.figure(figsize=(10,3))
# cty 1
x = [2016, 2017, 2018, 2019, 2020, 2021]
y = [45, 46, 48, 50, 52, 60]
# cty 2
x2 = [2016, 2017, 2018, 2019, 2020, 2021]
y2 = [41, 42, 44, 45, 52, 57]
plt.plot(x, y, marker='o', linestyle='-.', color='g', label='mongolia')
plt.plot(x2, y2, marker='d', linestyle='-', color='b', label='congo')
plt.xlabel('Year')
plt.ylabel('Population (M)')
plt.title('years vs population')
plt.legend(loc='upper center')
plt.yticks([45, 50, 55, 60])
plt.savefig('cty.png')
plt.show()

# In[76]:

#subplots
fig, ax = plt.subplots(1, 2, sharey=True) # rows and cols
ax[0].plot(x,y,color='r')
ax[1].plot(x2,y2,color='b')
plt.show()

# In[70]:

# Bar Plots
plt.figure(figsize=(8,3))
bar1 = ['asia', 'america', 'europe', 'oceania', 'africa']
bar2 = [30, 20, 20, 5, 25]
plt.bar(bar1, bar2)
plt.show()

# In[48]:

# pie
plt.pie(y, labels=x)
plt.show()

# In[57]:

# histogram
conts = ['asia', 'america', 'europe', 'oceania', 'africa']
pers = [30, 15, 25, 5, 25]
bins = [5,10,15,20,25,30,35]
plt.hist(pers,bins,edgecolor='black')
plt.show()

# In[69]:

# scatterplot
conts = ['asia', 'america', 'europe', 'oceania', 'africa']
scat1 = [30,15,25,5,25,50]
scat2 = [5,10,12,2,1,4]
plt.scatter(scat1,scat2)
plt.show()

# In[ ]:


def are_anagrams(str1, str2):
    if len(str1) != len(str2):
        return False

    char_count = {}

    # Подсчитываем частоту каждого символа в первой строке
    for char in str1:
        char_count[char] = char_count.get(char, 0) + 1

    # Уменьшаем частоту каждого символа во второй строке
    for char in str2:
        if char in char_count:
            char_count[char] -= 1
        else:
            return False

    # Проверяем, что все частоты стали равны нулю
    for count in char_count.values():
        if count != 0:
            return False

    return True

if __name__ == "__main__":
    str1 = input().strip()
    str2 = input().strip()

    result = 1 if are_anagrams(str1, str2) else 0
    print(result)


import numpy as np
import numpy.random as rd
import matplotlib.pyplot as plt
import seaborn as sns  # noqa
from itertools import islice

from ibm_rng import ibm_rng

from pylatex import Plt

N = 1000

def main():
    dist_to_tex('np_random.tex', 'Numpy RNG', *get_random_pairs(N).T)

    ibm_gen = ibm_rng(1)
    dist_to_tex('ibm_random.tex', 'IBM RNG',
                take_samples(ibm_gen), take_samples(ibm_gen))

def take_samples(gen, n=N):
    return np.array(list(islice(gen, n)))

def dist_to_tex(filename, caption, x, y):
    plot_distribution(x, y, show=False)
    with open(filename, 'w') as f:
        plot = Plt(position='htbp')
        plot.add_plot(plt)
        plot.add_caption(caption)

        plot.dump(f)

def plot_distribution(x, y, show=True):
    sns.jointplot(x, y, stat_func=None)
    if show:
        plt.show()

def get_random_pairs(n):
    return rd.rand(n, 2)

if __name__ == '__main__':
    main()


from aiogram.dispatcher.filters.state import State, StatesGroup
from aiogram.dispatcher import FSMContext
from aiogram import types
from aiogram.dispatcher.filters import Text

from main import dp, admins_id
from utils.db_api.db_quick_commands import add_contest

class FSMAdmin(StatesGroup):
    name = State()
    photo = State()
    description = State()
    price = State()

@dp.message_handler(state="*", text="Отмена")
@dp.message_handler(Text(equals="Отмена", ignore_case=True), state="*")
async def cancel_handler(message: types.Message, state: FSMContext):
    current_state = await state.get_state()
    if current_state is None:
        return
    await state.finish()
    await message.reply("Ввод отменен")

@dp.message_handler(content_types=types.ContentType.TEXT, text="Добавить конкурс")
async def start_command(message: types.Message, state: FSMContext):
    if str(message.from_user.id) in admins_id:
        await FSMAdmin.name.set()
        await message.reply("Введите название конкурса")

@dp.message_handler(state=FSMAdmin.name)
async def add_name(message: types.Message, state: FSMContext):
    if str(message.from_user.id) in admins_id:
        async with state.proxy() as data:
            data["name"] = message.text
        await FSMAdmin.next()
        await message.reply("Добавте фото конкурса")

@dp.message_handler(content_types=["photo"], state=FSMAdmin.photo)
async def add_photo(message: types.Message, state: FSMContext):
    if str(message.from_user.id) in admins_id:
        async with state.proxy() as data:
            data["photo"] = message.photo[0].file_id
        await FSMAdmin.next()
        await message.reply("Введите описание конкурса")

@dp.message_handler(state=FSMAdmin.description)
async def add_description(message: types.Message, state: FSMContext):
    if str(message.from_user.id) in admins_id:
        async with state.proxy() as data:
            data["description"] = message.text
        await FSMAdmin.next()
        await message.reply("Введите взнос")

@dp.message_handler(state=FSMAdmin.price)
async def add_price(message: types.Message, state: FSMContext):
    if str(message.from_user.id) in admins_id:
        async with state.proxy() as data:
            data["price"] = float(message.text.replace(",", "."))

        async with state.proxy() as data:
            if add_contest(data):
                await message.answer("Конкурс добавлен")
            else:
                await message.answer("Конкурс не добавлен.\nНе коррекртные данные")
        await state.finish()


import unittest
from neural_network import Neuron, Connection, NeuralNetwork, Innovations

class Test_test_1(unittest.TestCase):
    def test_network_size(self):
        self.global_innovations = Innovations(0, {}) # initialize global innovations object
        self.nn = NeuralNetwork(5, 0, 5, [], [], self.global_innovations)

        self.assertTrue(self.nn.network_size() == 10)

    def test_weight_mutation(self):
        self.n_in = Neuron(1.0, [], [], 1, 1)
        self.n_out = Neuron(0, [], [], 3, 2)

        self.test_conn = Connection(self.n_in, self.n_out, 0, 1)

        self.assertTrue(-1.0 <= self.test_conn.weight <= 1.0)
        self.test_conn.mutate_weight()
        self.assertTrue(-1.0 <= self.test_conn.weight <= 1.0)

if __name__ == '__main__':
    unittest.main()


x= int(input("Enter the first number"))
y= int(input("Enter the secomd number"))

def lcm(num1,num2):
     if num1>num2:
         gratter=num1
     else:
         gratter=num2

     while(True):
        if(gratter%num1==0) and (gratter%num2==0):
            ans= gratter
            break
        gratter +=1

     return ans

print(f"LCM of the number {x,y} is", lcm(x,y))


import pandas as pd
import numpy as np
from matplotlib.colors import ListedColormap
import seaborn as sns
import matplotlib.pyplot as plt
sns.set_style("darkgrid", {'font.family':'serif'})
plt.style.use("seaborn-darkgrid")
color = ['#fccbcb','#F99797', '#F76C6C', '#ABD0E6', '#23305E']
cmap = ListedColormap(color)
color2 = ['#90113f', '#afb5c0', '#e39292', '#61749f', '#b4d5ea', '#c26f6d']

#====================================== Data Airbnb Singapore ============================================
df = pd.read_csv('data-airbnbsg.csv')

host_features = ['total_price','host_is_superhost','scores_rating', 'host_response_rate','scores_communication', 'scores_cleanliness','scores_checkin' , 'scores_communication','scores_location', 'number_of_reviews', 'reviews_per_month']
listings_features = ['total_price', 'number_of_reviews', 'bathrooms', 'bedrooms', 'beds', 'accommodates', 'area','region','property_type', 'room_type', 'scores_location', 'scores_cleanliness', 'scores_rating']

# Property type data  --------------------------------------------------------------------------------
a = df['property_type'].value_counts().reset_index()
fig = plt.figure(figsize=(10,10))
ax =sns.barplot(y='index', x='property_type', data=a, palette=color)
plt.xlabel('Property Type')
plt.ylabel('Count')
plt.title('Airbnb Listings by Property Type', fontsize=15, weight='bold')
for p in ax.patches:
    ax.text(p.get_width()+50, p.get_height()/2 + p.get_y(), int(p.get_width()) ,ha="center")
plt.show()
plt.close()

a['persen'] = a['property_type'] / len(df) *100
# print(a)
a = a[a['persen'] > 4]['index']
a = a.tolist()
df = df[df['property_type'].isin(a)]

#================================================= Exploring the Data =================================================

# Figure : Listings by Region
a = df['region'].value_counts().reset_index().sort_values(by='index').reset_index(drop=True)
f, ax = plt.subplots(ncols=2, nrows=1, figsize=(12, 8))
sns.barplot(y='index', x='region', data=a, palette=color, ax=ax[0])
ax[0].set(xlabel='Count of Property Type', ylabel='')
ax[0].set_title('Airbnb Listings by Region', fontsize=15, weight='bold')
numListing =[]
for p in ax[0].patches:
    ax[0].text(p.get_width()+170, p.get_height()/2 + p.get_y(), int(p.get_width()), ha="center")
    numListing.append(int(p.get_width()))
# Figure : Bookings by Region
a = df.groupby(by=['region'], as_index=False).number_of_reviews.sum()
sns.barplot(y='region', x='number_of_reviews', data=a, palette=color, ax=ax[1])
ax[1].set(xlabel='Numbers of Booking', ylabel='')
ax[1].set_title('Airbnb Bookings in Singapore', fontsize=15, weight='bold')
for p in ax[1].patches:
    ax[1].text(p.get_width()+2600, p.get_height()/2 + p.get_y(), int(p.get_width()) ,ha="center")
plt.tight_layout()
plt.show()
a['numListing'] = numListing
a['supply-demand'] = numListing/a['number_of_reviews'] *100
print(a)

# Figure : Number of Listings by sub-district in Singapore ----------------------------------------
f, ax = plt.subplots(ncols=2, nrows=1, figsize=(10, 6))
a = df.groupby(['area', 'property_type']).size().unstack()
# print(a)
a.plot(kind='barh', stacked=True, ax=ax[0], color=color)
ax[0].set_title('Number of Listings by Subdistrict', fontsize=12)
ax[0].set(xlabel='Count of Property', ylabel='')
ax[0].yaxis.grid(True)
ax[0].legend()
# Figure : Number of Property_type ----------------------------------------------------------------------
a = df['property_type'].value_counts().reset_index().sort_values(by='index').reset_index(drop=True)
sns.barplot(
    x='property_type', y='index', data=a,
    label="Property type", palette=color, ax= ax[1])
ax[1].set(xlabel='Count of Property Type', ylabel='')
ax[1].set_title('Number of Listings by Property', fontsize=12)
ax[1].yaxis.grid(True)
plt.suptitle('Airbnb in Central Singapore Area', fontsize=18, weight='bold')
plt.subplots_adjust(top=.88, wspace=0.3)
plt.show()
plt.close()

# Figure : Property_type vs price ---------------------------------------------------------------------
f, ax = plt.subplots(figsize=(10, 8), ncols=2, nrows=1)
ax[0].set_xscale("log")

sns.boxplot(x="total_price", y="property_type", data=df,
            whis="range", palette=color, ax=ax[0])

ax[0].set_title("Price by Property Type", fontsize=15, weight='bold')
ax[0].xaxis.grid(True)
ax[0].set(xlabel="Price",ylabel="")
# Figure : Room_type vs price -----------------------------------------------------------------------------
ax[1].set_xscale("log")

sns.boxplot(x="total_price", y="room_type", data=df,
            whis="range", palette=color, ax=ax[1])

ax[1].set_title("Price by Room Type", fontsize=15, weight='bold')
ax[1].xaxis.grid(True)
ax[1].set(xlabel="Price",ylabel="")
plt.tight_layout()
plt.show()
plt.close()

# Figure : Price per guest ------------------------------------------------------------------------------
f, ax = plt.subplots(figsize=(14, 10))

sns.heatmap(df.groupby([
        'area', 'accommodates']).total_price.median().unstack(),
            annot=True,
            fmt=".0f",
           cmap=cmap)
ax.set(xlabel="Maximum Number of Guest",ylabel="")
ax.set_title('Price (USD) per Guest', fontsize=15, weight='bold')
plt.show()
plt.close()
# Figure : Area vs price -----------------------------------------------------------------------------
df.plot(kind="scatter", x="longitude", y="latitude", alpha=0.4,c="total_price", cmap=plt.get_cmap("gnuplot2_r"), colorbar=True, figsize=(10,8))
plt.title('Airbnb Singapore Price', fontsize=12)
# plt.show()
plt.close()
# Figure : Area vs price -----------------------------------------------------------------------------
fig, ax = plt.subplots(figsize = (10,8))
sns.boxplot(x="area", y="total_price", data=df,
            whis="range", palette=color)
ax.set_yscale("log")
ax.set_title("Price by Area", fontsize=15, weight='bold')
ax.xaxis.grid(True)
ax.set(xlabel="",ylabel="Price")
ax.set_xticklabels(ax.get_xticklabels(), rotation=90)
plt.tight_layout()
plt.subplots_adjust(bottom= .28)
plt.show()
plt.close()

# Figure : Number of Super Host Listings  ----------------------------------------------------------------
f, ax = plt.subplots(figsize=(10, 8))
a = df['area'].value_counts().reset_index()
sns.barplot(
    x='area', y='index', data=a,
    label="Total Listings", color=color[0])

a = df['area'][df['host_is_superhost'] == 1].value_counts().reset_index()

sns.barplot(x="area", y="index", data=a,
            label="Superhost Listings", color=color[-1])

ax.legend(ncol=2, loc="best", frameon=True)
ax.set(ylabel="", xlabel="Airbnb Listings")
sns.despine(left=True, bottom=True)
ax.set_title("Number of Superhost Listings in Central Singapore", fontsize=15, weight='bold')
for p in ax.patches:
    ax.text(p.get_width()+23, p.get_height()/2 + p.get_y(), int(p.get_width()) ,ha="center")
plt.show()
plt.close()

# Correlation Price - Host Features ----------------------------------------------------------------------------

f, ax = plt.subplots(figsize=(12, 10))
corr = df[host_features].corr(method= 'spearman')
sns.heatmap(corr, cmap= cmap, annot=True, fmt=".2f")
ax.set_xticklabels(ax.get_xticklabels())
plt.tight_layout()
plt.subplots_adjust(bottom= .25)
plt.show()
plt.close()

# Correlation Price - Listings Features ------------------------------------------------------------------------
f, ax = plt.subplots(figsize=(12, 10))
corr = df[listings_features].corr(method= 'spearman')
sns.heatmap(corr, cmap= cmap, annot=True, fmt=".2f")
ax.set_xticklabels(ax.get_xticklabels())
plt.tight_layout()
plt.subplots_adjust(bottom= .21)
plt.show()
plt.close()

# save to csv
df.to_csv('data-airbnbsg_cleaned.csv', header=True, index=False)


from PyQt6.QtWidgets import QWidget, QHBoxLayout, QVBoxLayout, QGridLayout, QPushButton, QLabel, QSizePolicy, \
    QStyleOptionButton, QStylePainter, QColorDialog, QMessageBox, QApplication, QFrame
from PyQt6.QtGui import QColor, QPalette, QPainterPath, QPainter, QPen, QBrush, QLinearGradient, QRadialGradient
from PyQt6.QtCore import Qt, QSettings, QPointF

from CustomStyle import MyChangingPalette
from DatabaseprojectGUI import MainWindow
from UniversalFunctions import get_signal
from MyWidgets import MyColorDialog

class EditThemes(QWidget):

    def __init__(self, window):
        super().__init__()
        # Stored Variables
        self.window = window

        self.setWindowTitle("Themes")
        self.setFixedSize(900, 700)

        self.cur_theme = None

        self.temp_background = QColor()
        self.temp_text = QColor()
        self.temp_highlight = QColor()

        # Font
        font = QLabel().font()
        font.setUnderline(True)

        # labels
        themes_label = QLabel("Themes:")
        themes_label.setAlignment(Qt.AlignmentFlag.AlignCenter)
        themes_label.setFont(font)
        dark_label = QLabel("Dark")
        dark_label.setAlignment(Qt.AlignmentFlag.AlignCenter)
        light_label = QLabel("Light")
        light_label.setAlignment(Qt.AlignmentFlag.AlignCenter)
        custom_themes_label = QLabel("Custom Themes:")
        custom_themes_label.setAlignment(Qt.AlignmentFlag.AlignCenter)
        custom_themes_label.setFont(font)
        color_options_label = QLabel("Color Options:")
        color_options_label.setAlignment(Qt.AlignmentFlag.AlignCenter)
        color_options_label.setFont(font)
        background_label = QLabel("Background")
        background_label.setAlignment(Qt.AlignmentFlag.AlignCenter)
        text_label = QLabel("Text")
        text_label.setAlignment(Qt.AlignmentFlag.AlignCenter)
        highlight_label = QLabel("Highlight")
        highlight_label.setAlignment(Qt.AlignmentFlag.AlignCenter)

        # Palette and Preview Window
        self.preview_palette = MyChangingPalette(preview=True)

        self.window_preview = MainWindow()
        self.window_preview.setAutoFillBackground(True)
        self.window_preview.setFixedSize(600, 300)
        self.window_preview.setDisabled(True)
        self.window_preview.setPalette(self.preview_palette)

        self.background_button = EditColorButton(self)
        self.text_button = EditColorButton(self)
        self.highlight_button = EditColorButton(self)

        self.dark_theme = ThemeButton(self)
        self.light_theme = ThemeButton(self, True)
        self.custom_themes = [ThemeButton(self), ThemeButton(self), ThemeButton(self),
                              ThemeButton(self, True), ThemeButton(self, True), ThemeButton(self, True)]

        # Custom Color Dialog
        self.dialog = MyColorDialog()
        self.dialog.setOption(QColorDialog.ColorDialogOption.DontUseNativeDialog, True)

        # Buttons
        apply = QPushButton("&Apply")
        apply.setFixedWidth(108)
        apply.pressed.connect(self.apply_palette)

        cancel = QPushButton("Cancel")
        cancel.setFixedWidth(108)
        cancel.pressed.connect(self.load_from_temps)

        restore_defaults = QPushButton("&Reset Themes")
        restore_defaults.setFixedWidth(108)
        restore_defaults.pressed.connect(self.load_default_themes)

        # Reading Saved Themes
        try:
            self.readThemes()
        except:
            pass

        # setting initial selection
        self.dark_theme.set_preview()

        # Spacer/Seperator Objects
        self.spacer = QWidget()
        self.spacer.setMaximumHeight(170)
        self.spacer.setSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Expanding)

        spacer = QWidget()
        spacer.setFixedHeight(250)
        spacer.setSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Expanding)

        s = QWidget()
        s.setFixedHeight(20)
        s.setSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Expanding)

        seperator = QFrame()
        seperator.setFrameShape(QFrame.Shape.VLine)
        seperator.setLineWidth(1)

        # setting layouts in order of depth (top left to bottom right ish)
        default_themes_layout = QGridLayout()
        default_themes_layout.addWidget(dark_label, 1, 1)
        default_themes_layout.addWidget(light_label, 1, 2)
        default_themes_layout.addWidget(self.dark_theme, 2, 1)
        default_themes_layout.addWidget(self.light_theme, 2, 2)

        custom_themes_layout = QGridLayout()
        for i, theme in enumerate(self.custom_themes):
            custom_themes_layout.addWidget(theme, i // 3, i - 3 if i > 2 else i)

        themes_layout = QVBoxLayout()
        themes_layout.setAlignment(Qt.AlignmentFlag.AlignHCenter)
        themes_layout.addWidget(self.spacer)
        themes_layout.addWidget(themes_label)
        themes_layout.addLayout(default_themes_layout)
        themes_layout.addWidget(custom_themes_label)
        themes_layout.addLayout(custom_themes_layout)
        themes_layout.addWidget(s)
        themes_layout.addWidget(restore_defaults)
        themes_layout.setAlignment(restore_defaults, Qt.AlignmentFlag.AlignCenter)
        themes_layout.addWidget(spacer)

        colors_layout = QGridLayout()
        colors_layout.setHorizontalSpacing(50)
        colors_layout.setVerticalSpacing(8)
        colors_layout.setAlignment(Qt.AlignmentFlag.AlignHCenter)
        colors_layout.addWidget(background_label, 0, 1)
        colors_layout.addWidget(text_label, 0, 2)
        colors_layout.addWidget(highlight_label, 0, 3)
        colors_layout.addWidget(self.background_button, 1, 1)
        colors_layout.setAlignment(self.background_button, Qt.AlignmentFlag.AlignCenter)
        colors_layout.addWidget(self.text_button, 1, 2)
        colors_layout.setAlignment(self.text_button, Qt.AlignmentFlag.AlignCenter)
        colors_layout.addWidget(self.highlight_button, 1, 3)
        colors_layout.setAlignment(self.highlight_button, Qt.AlignmentFlag.AlignCenter)

        button_layout = QHBoxLayout()
        button_layout.addWidget(apply)
        button_layout.addWidget(cancel)

        value_layout = QVBoxLayout()
        value_layout.addWidget(s)
        value_layout.addWidget(self.dialog.color_shower)
        value_layout.addWidget(s)
        value_layout.addLayout(button_layout)

        dialog_layout = QHBoxLayout()
        dialog_layout.addWidget(self.dialog)
        dialog_layout.setAlignment(self.dialog, Qt.AlignmentFlag.AlignBaseline)
        dialog_layout.addLayout(value_layout)

        preview_and_colors_layout = QVBoxLayout()
        preview_and_colors_layout.setAlignment(Qt.AlignmentFlag.AlignTop)
        preview_and_colors_layout.addWidget(self.window_preview)
        preview_and_colors_layout.addWidget(color_options_label)
        preview_and_colors_layout.addLayout(colors_layout)
        preview_and_colors_layout.addLayout(dialog_layout)

        layout = QHBoxLayout()
        layout.addLayout(themes_layout)
        layout.addWidget(spacer)
        layout.addWidget(seperator)
        layout.addLayout(preview_and_colors_layout)

        self.setLayout(layout)

    # Handles changing colors in preview and color options
    def show_color_options(self, theme):
        self.set_palette_colors(theme, self.preview_palette)
        self.window_preview.table.horizontalHeader().sectionPressed.emit(0)

        b, t, h = theme.getColors()

        self.background_button.setTheme(theme, b)
        self.text_button.setTheme(theme, t)
        self.highlight_button.setTheme(theme, h)
        # handling exception for default palettes
        if theme in (self.dark_theme, self.light_theme):
            self.dialog.blockSignals(True)
        else:
            self.dialog.blockSignals(False)
        self.background_button.connectToDialog()
        self.cur_theme = theme

    def set_palette_colors(self, theme, palette):
        palette.setShadeColors(theme.background_color)
        palette.setBackgroundColors(theme.background_color)
        palette.setHighlightColor(theme.highlight_color)
        palette.setTextColor(theme.text_color)
        self.window_preview.setPalette(self.preview_palette)

    def apply_palette(self):
        if self.window.current_database is not None and self.window.databases[self.window.current_database].skip is True:
            self.window.databases[self.window.current_database].thread.quit()

        if self.window_preview.current_database is not None and self.window_preview.databases[self.window_preview.current_database].skip is True:
            self.window_preview.databases[self.window_preview.current_database].thread.quit()
        settings = QSettings(r'settings.ini', QSettings.Format.IniFormat)
        settings.beginGroup("themes")
        if self.window.databases[self.window.current_database].skip is True:
            self.window.databases[self.window.current_database].thread.quit()
        self.window.databases[self.window.current_database].thread = None
        self.window.databases[self.window.current_database].worker = None
        settings.setValue("palette", [self.preview_palette.background, self.preview_palette.text,
                                      self.preview_palette.highlight])
        settings.endGroup()
        self.set_palette_colors(self.cur_theme, self.preview_palette)
        self.set_palette_colors(self.cur_theme, QApplication.style().changing_palette)
        QApplication.style().update_palette()
        self.window_preview.repaint()

    def store_temps(self, theme):
        for color, temp in zip(theme.getColors(), [self.temp_background, self.temp_text, self.temp_highlight]):
            r, g, b, a = color.getRgb()
            temp.setRgb(r, g, b, a)

    def load_from_temps(self):
        for color, temp in zip(self.cur_theme.getColors(), [self.temp_background, self.temp_text, self.temp_highlight]):
            r, g, b, a = temp.getRgb()
            color.setRgb(r, g, b, a)
        self.show_color_options(self.cur_theme)
        self.repaint()
        self.apply_palette()

    def saveThemes(self):
        settings = QSettings(r'settings.ini', QSettings.Format.IniFormat)
        settings.beginGroup("themes")
        settings.setValue("customthemes", [[theme.background_color, theme.text_color, theme.highlight_color]
                                           for theme in self.custom_themes])
        settings.endGroup()

    def readThemes(self):
        settings = QSettings(r'settings.ini', QSettings.Format.IniFormat)
        settings.beginGroup("themes")
        themes = settings.value("customthemes")
        for colors, theme in zip(themes, self.custom_themes):
            r, g, b, a = colors[0].getRgb()
            theme.background_color.setRgb(r, g, b, a)
            theme.text_color = colors[1]
            theme.highlight_color = colors[2]
        settings.endGroup()

    def load_default_themes(self):
        if self.cur_theme:
            ret = QMessageBox.question(self, "Reset Themes",
                                       "Would you like to reset themes to default?",
                                       QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.Cancel)
            if ret == QMessageBox.StandardButton.Cancel:
                return

        for theme in self.custom_themes[:3]:
            theme.set_dark()
            theme.repaint()

        for theme in self.custom_themes[3:]:
            theme.set_light()
            theme.repaint()

        self.dark_theme.set_preview()

    def closeEvent(self, event) -> None:
        ret = QMessageBox.question(self, "Save",
                                   "Would you like to save themes before closing?",
                                   QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No |
                                   QMessageBox.StandardButton.Cancel)
        if ret == QMessageBox.StandardButton.Cancel:
            event.ignore()
            return
        elif ret == QMessageBox.StandardButton.Yes:
            self.saveThemes()
        self.window.setEnabled(True)
        event.accept()

class ColorButton(QPushButton):

    def __init__(self, color=QColor(Qt.GlobalColor.black)):
        super().__init__()
        self.setFixedSize(50, 50)
        self.check = False

        self.button_color = color

    def paintEvent(self, a0) -> None:
        option = QStyleOptionButton()
        self.initStyleOption(option)
        painter = QStylePainter(self)

        fill = QRadialGradient(QPointF(self.rect().center()), 25, QPointF(self.rect().center()))
        fill.setColorAt(0.0, self.button_color)
        fill.setColorAt(1.0, self.button_color.lighter(110))
        path = self.draw_circle(option.rect)
        path.setFillRule(Qt.FillRule.WindingFill)

        painter.setRenderHint(QPainter.RenderHint.Antialiasing, True)
        painter.fillPath(path, fill)

        painter.save()
        pen = QPen(Qt.GlobalColor.black, 2)
        if self.check:
            pen = QPen(Qt.GlobalColor.darkRed, 2)
        painter.setPen(pen)
        painter.drawPath(path)

        painter.restore()

    @staticmethod
    def draw_circle(rect, highlight=False):
        x1, y1, x2, y2 = rect.getRect()
        radius = (x2 / 2) - 1.5
        if highlight:
            radius = (x2 / 2) - 3

        path = QPainterPath()
        path.addRoundedRect(x1 + 1, y1 + 1, x2 - 2, y2 - 2, radius, radius, Qt.SizeMode.RelativeSize)

        return path

class ThemeButton(ColorButton):

    def __init__(self, window, light=False):
        self.win = window

        self.background_color = QColor()
        self.text_color = QColor()
        self.highlight_color = QColor()

        if light:
            self.set_light()
        else:
            self.set_dark()
        super().__init__(self.background_color)

        self.clicked.connect(self.set_preview)

    def set_light(self):
        self.background_color.setRgb(240, 240, 250)
        self.text_color.setRgb(0, 0, 0)
        self.highlight_color.setRgb(0, 144, 238)

    def set_dark(self):
        self.background_color.setRgb(50, 50, 58)
        self.text_color.setRgb(240, 238, 243)
        self.highlight_color.setRgb(70, 80, 180)

    def set_preview(self):
        for button in self.win.custom_themes + [self.win.dark_theme, self.win.light_theme]:  # changes visual drawn
            button.check = False
            button.repaint()
        self.check = True
        self.repaint()
        self.win.store_temps(self)
        self.win.show_color_options(self)

    def getColors(self):
        return self.background_color, self.text_color, self.highlight_color

class EditColorButton(ColorButton):

    def __init__(self, window):
        super().__init__()
        self.win = window
        self.clicked.connect(self.connectToDialog)
        self.cur_theme = None

    def set_color(self, color):
        r, g, b, a = color.getRgb()
        self.button_color.setRgb(r, g, b, a)
        self.repaint()
        self.cur_theme.repaint()
        self.win.set_palette_colors(self.cur_theme, self.win.preview_palette)

    def setTheme(self, theme, color):
        self.cur_theme = theme
        self.button_color = color
        self.repaint()

    def connectToDialog(self):
        for button in [self.win.background_button, self.win.text_button, self.win.highlight_button]:  # changes visual
            button.check = False
            button.repaint()
        self.check = True
        self.repaint()
        if self.win.dialog.isSignalConnected(get_signal(self.win.dialog, "currentColorChanged")):
            self.win.dialog.currentColorChanged.disconnect()
        self.test()
        self.win.dialog.setCurrentColor(self.button_color)

    def test(self):
        self.win.dialog.currentColorChanged.connect(self.set_color)


# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right
class Solution:
    def kthSmallest(self, root: Optional[TreeNode], k: int) -> int:

        ans = -1

        def inorder(node):
            nonlocal k
            if not node or k == 0:
                return

            inorder(node.left)
            k -= 1
            nonlocal ans
            if k == 0: ans = node.val
            inorder(node.right)

        inorder(root)
        return ans


"""Module to describe an atomic structure."""
import json
from jarvis.db.figshare import get_jid_data
from jarvis.core.atoms import Atoms
from jarvis.analysis.structure.spacegroup import Spacegroup3D
from jarvis.analysis.diffraction.xrd import XRD
from jarvis.core.specie import Specie
import pprint
from collections import defaultdict

def atoms_describer(
    atoms=[], xrd_peaks=5, xrd_round=1, cutoff=4, take_n_bomds=2
):
    """Describe an atomic structure."""
    spg = Spacegroup3D(atoms)
    theta, d_hkls, intens = XRD().simulate(atoms=(atoms))
    #     x = atoms.atomwise_angle_and_radial_distribution()
    #     bond_distances = {}
    #     for i, j in x[-1]["different_bond"].items():
    #         bond_distances[i.replace("_", "-")] = ", ".join(
    #             map(str, (sorted(list(set([round(jj, 2) for jj in j])))))
    #         )
    dists = defaultdict(list)
    elements = atoms.elements
    for i in atoms.get_all_neighbors(r=cutoff):
        for j in i:
            key = "-".join(sorted([elements[j[0]], elements[j[1]]]))
            dists[key].append(j[2])
    bond_distances = {}
    for i, j in dists.items():
        dist = sorted(set([round(k, 2) for k in j]))
        if len(dist) >= take_n_bomds:
            dist = dist[0:take_n_bomds]
        bond_distances[i] = ", ".join(map(str, dist))
    fracs = {}
    for i, j in (atoms.composition.atomic_fraction).items():
        fracs[i] = round(j, 3)
    info = {}
    chem_info = {
        "atomic_formula": atoms.composition.reduced_formula,
        "prototype": atoms.composition.prototype,
        "molecular_weight": round(atoms.composition.weight / 2, 2),
        "atomic_fraction": json.dumps(fracs),
        "atomic_X": ", ".join(
            map(str, [Specie(s).X for s in atoms.uniq_species])
        ),
        "atomic_Z": ", ".join(
            map(str, [Specie(s).Z for s in atoms.uniq_species])
        ),
    }
    struct_info = {
        "lattice_parameters": ", ".join(
            map(str, [round(j, 2) for j in atoms.lattice.abc])
        ),
        "lattice_angles": ", ".join(
            map(str, [round(j, 2) for j in atoms.lattice.angles])
        ),
        "spg_number": spg.space_group_number,
        "spg_symbol": spg.space_group_symbol,
        "top_k_xrd_peaks": ", ".join(
            map(
                str,
                sorted(list(set([round(i, xrd_round) for i in theta])))[
                    0:xrd_peaks
                ],
            )
        ),
        "density": round(atoms.density, 3),
        "crystal_system": spg.crystal_system,
        "point_group": spg.point_group_symbol,
        "wyckoff": ", ".join(list(set(spg._dataset["wyckoffs"]))),
        "bond_distances": bond_distances,
        "natoms_primitive": spg.primitive_atoms.num_atoms,
        "natoms_conventional": spg.conventional_standard_structure.num_atoms,
    }
    info["chemical_info"] = chem_info
    info["structure_info"] = struct_info
    return info

if __name__ == "__main__":
    atoms = Atoms.from_dict(
        get_jid_data(jid="JVASP-32", dataset="dft_3d")["atoms"]
    )
    info = atoms_describer(atoms=atoms)
    pprint.pprint(info)
    from pygments import highlight
    from pygments.formatters.terminal256 import Terminal256Formatter
    from pygments.lexers.web import JsonLexer

    raw_json = json.dumps(info, indent=4)
    colorful = highlight(
        raw_json,
        lexer=JsonLexer(),
        formatter=Terminal256Formatter(),
    )
    print(colorful)


import pytest
from removeDuplicates import Solution

@pytest.mark.parametrize("nums,expected", [
    ([1, 2, 2, 3], [1, 2, 2, 3]),
    ([1, 1, 1, 2, 2, 3], [1, 1, 2, 2, 3]),
    ([1, 1, 1, 2, 2, 3, 3, 3, 3], [1, 1, 2, 2, 3, 3]),
    ([0, 0, 1, 1, 1, 1, 2, 3, 3], [0, 0, 1, 1, 2, 3, 3])
])
def test_removeDuplicates(nums, expected):
    length = Solution().removeDuplicates(nums)
    assert nums == expected
    assert len(nums) == length


'''
Module to define transformations on dataframes
'''
import pyspark.sql.functions as F

def transform_data(input_df):
    ''' This function takes a dataframe as input and returns a transformed
    dataframe'''
    transformed_df = (input_df
                      .groupBy('Location',)
                      .agg(F.sum('ItemCount').alias('TotalItemCount')))
    return transformed_df


from typing import List
from fastapi import APIRouter

from schemas.season import Season

router = APIRouter()

@router.get("/seasons", response_model=List[Season])
async def get_seasons():
    return True

@router.post("/seasons")
async def create_season(season: Season):
    db["seasons"].append(season)
    return True

@router.get("/seasons/{id}")
async def get_season():
    return True

@router.get("/seasons/{id}/episodes")
async def get_episodes_from_season():
    return True


"""The services tests module."""
import pytest
from django.conf import settings
from django.core.exceptions import ValidationError
from django.db.models.query import QuerySet

from communication.models import Review, ReviewComment
from communication.validators import (validate_message_parent,
                                      validate_message_recipient,
                                      validate_reminder_remind_before,
                                      validate_review_comment_user,
                                      validate_review_user)
from users.models import User

pytestmark = pytest.mark.django_db

def test_validate_reminder_remind_before():
    """Should validate remind_before field."""
    step: int = settings.D8B_REMINDER_INTERVAL
    with pytest.raises(ValidationError):
        validate_reminder_remind_before(int(step * 1.5))
    validate_reminder_remind_before(step * 3)

def test_validate_review_comment_user(admin: User, reviews: QuerySet):
    """Should validate the review comment user."""
    comment = ReviewComment()
    comment.user = admin
    comment.review = reviews.exclude(professional__user=admin).first()

    with pytest.raises(ValidationError):
        validate_review_comment_user(comment)

def test_validate_review_user(admin: User, professionals: QuerySet):
    """Should validate the review user."""
    review = Review()
    review.user = admin
    review.professional = professionals.filter(user=admin).first()
    review.rating = 3  # type: ignore
    review.description = "description"

    with pytest.raises(ValidationError):
        validate_review_user(review)

def test_validate_message_recipient(admin: User, messages: QuerySet):
    """Should mark the message read."""
    message = messages.filter(sender=admin).first()
    validate_message_recipient(message)
    message.recipient = admin
    with pytest.raises(ValidationError):
        validate_message_recipient(message)

def test_validate_parent(admin: User, messages: QuerySet):
    """Should validate the message parent message."""
    messages = messages.filter(sender=admin)
    message = messages[0]
    message.parent = messages[1]
    with pytest.raises(ValidationError):
        validate_message_parent(message)
    message.is_read = True
    with pytest.raises(ValidationError):
        validate_message_parent(message)
    message.parent = messages.filter(recipient=admin).first()
    validate_message_parent(message)


import pygame
from pygame.locals import *

import const

class LineC():
    def lf(self):
        Line.pos += const.LINEHEIGHT

    def menu_lf(self):
        Line.pos += const.MENUHEIGHT

    def output(self, text, color, font, lf):
        label = font.render(text, True, color)
        text_rect = label.get_rect(midtop=const.SCREENRECT.midtop)
        text_rect.move_ip(0,Line.pos)
        self.screen.blit(label,text_rect)
        lf()

    def big(self, text, color = (0,0,0,255)):
        self.output(text, color, const.font.other, self.menu_lf)

    def small(self, text, color = (0,0,0,255)):
        self.output(text, color, const.font.small, self.lf)

Line = LineC()

class MenuEntry(pygame.sprite.Sprite):
    def __init__(self,label,result,width):
        pygame.sprite.Sprite.__init__(self,self.containers)

        self.image = pygame.Surface((width,const.MENUHEIGHT),flags=SRCALPHA).convert_alpha()
        self.image.fill((255, 255, 255, 0))

        self.rect = self.image.get_rect(midtop=const.SCREENRECT.midtop)

        self.label = const.font.menu.render(label, True, (0,0,0,255))
        self.label_rect = self.label.get_rect(center=self.rect.center)

        self.rect.move_ip(0,Line.pos)

        Line.menu_lf()

        self.result   = result

        self.prev = MenuEntry.prev
        MenuEntry.prev = self
        if self.prev:
            self.prev.next = self
        self.next = None

    def update(self,selected):
        if selected:
            alpha = 125
        else:
            alpha = 0
        self.image.fill((0, 0, 0, alpha))
        self.image.blit(self.label, self.label_rect)

def menu(screen):
    background = pygame.Surface(screen.get_size()).convert()
    # imgbg = load_image('background-menu.png')
    background.fill((250, 250, 250))
    #background.blit(imgbg,(0,0))
    screen.blit(background, (0,0))
    pygame.display.flip()

    # Groups of sprite
    entries = pygame.sprite.Group()
    selected_entry = pygame.sprite.GroupSingle()
    visible = pygame.sprite.RenderUpdates()

    MenuEntry.containers = visible, entries
    MenuEntry.prev = None
    Line.pos = 0
    Line.screen = screen

    Line.lf()
    Line.big("Move Them",color = (22,22,90))
    Line.small("You are alone")
    Line.small("They want you")
    Line.small("And this is not a cannon you have")
    Line.lf()
    Line.lf()
    Line.lf()
    Line.small("Use arrow key to control, space to lock")
    Line.small("or a pad (A to lock)")
    Line.lf()

    pygame.display.flip()

    selected_entry.add(MenuEntry("Play",'play',screen.get_width()))
    MenuEntry("Highscore",'score',screen.get_width())
    MenuEntry("Quit",'quit',screen.get_width())

    # a clock
    clock = pygame.time.Clock()

    def noop(): pass
    def next():
        if selected_entry.sprite.next:
            selected_entry.add(selected_entry.sprite.next)
    def prev():
        if selected_entry.sprite.prev:
            selected_entry.add(selected_entry.sprite.prev)

    while True:
        nextaction = noop
        for event in pygame.event.get():
            if event.type == QUIT or \
                (event.type == KEYDOWN and event.key == K_ESCAPE):
                return 'quit'
            elif event.type == KEYDOWN and event.key == K_RETURN:
                return selected_entry.sprite.result
            elif event.type == KEYDOWN and event.key == K_UP:
                nextaction = prev
            elif event.type == KEYDOWN and event.key == K_DOWN:
                nextaction = next
            elif event.type == JOYAXISMOTION and event.joy == 0 and event.axis == 1:
                if event.value > 0.5:
                    nextaction = next
                elif event.value < -0.5:
                    nextaction = prev
            elif event.type == JOYBUTTONDOWN and event.joy == 0 and event.button == 0:
                return selected_entry.sprite.result
            elif event.type == JOYBUTTONDOWN and event.joy == 0 and (event.button == 2 or event.button == 6):
                return 'quit'

        nextaction()

        visible.clear(screen, background)

        visible.update(False)

        selected_entry.update(True)

        dirty = visible.draw(screen)
        pygame.display.update(dirty)

        #cap the framerate
        clock.tick(5)