"""
Functions and classes for sampling the *showq* output.
"""

import remote
from script     import Data_jobscript
from cfg        import Cfg
from qstatx     import Data_qstat
from sar        import Data_sar
from titleline  import title_line
import          rules
from mycollections import OrderedDict,od_add_list_item,od_last
from cluster    import current_cluster,cluster_properties

import pickle,os,shutil,gzip
from time       import sleep
import datetime

# list of users we want to ignore for the time being...
ignore_users = []

#===============================================================================
def run_showq():
    """
    1. Run command ``showq -r -p hopper --xml`` on a login node,
    2. Parse its xml output into an OrderedDict (xmltodict).

    Typical output of ``print(run_showq())`` ::

        OrderedDict([('Data', OrderedDict(
            [ ('Object', 'queue')
            , ('cluster', OrderedDict(
                            [ ('@LocalActiveNodes', '167')
                            , ('@LocalAllocProcs', '3305')
                            , ('@LocalConfigNodes', '168')
                            , ('@LocalIdleNodes', '1')
                            , ('@LocalIdleProcs', '54')
                            , ('@LocalUpNodes', '168')
                            , ('@LocalUpProcs', '3360')
                            , ('@RemoteActiveNodes', '0')
                            , ('@RemoteAllocProcs', '0')
                            , ('@RemoteConfigNodes', '0')
                            , ('@RemoteIdleNodes', '0')
                            , ('@RemoteIdleProcs', '0')
                            , ('@RemoteUpNodes', '0')
                            , ('@RemoteUpProcs', '0')
                            , ('@time', '1487065534')
                            ]))
            , ('queue', OrderedDict([ ('@count', '40')
                                    , ('@option', 'active')
                                    , ('job', [ <job entry OrderedDict for each job> ])
                                    ]))
            ]))
        ])

    """
    data_showq = remote.run("showq -r -p hopper --xml",post_processor=remote.xml_to_odict)
    return data_showq

#===============================================================================
class ShowqJobEntry:
    """
    Class for storing and manipulating a single job entry in the xml output of showq.

    Here is a typical job entry (in xml). It is converted to an :class:`OrderdDict`
    by :func:`xmltodict.parse`:

    .. code-block:: html

       <job
       AWDuration="192260"
       Class="q7d"
       DRMJID="393684.hopper"
       EEDuration="106250"
       GJID="393684"
       Group="vsc20213"
       JobID="393684"
       JobName="H2Diss-slabal2o3_MoO3"
       MasterHost="r3c4cn02.hopper.antwerpen.vsc"
       PAL="hopper"
       ReqAWDuration="604800"
       ReqProcs="160"
       RsvStartTime="1485577392"
       RunPriority="4970"
       StartPriority="4970"
       StartTime="1485577392"
       StatPSDed="30761912.000000"
       StatPSUtl="3691030.208000"
       State="Running"
       SubmissionTime="1485471138"
       SuspendDuration="0"
       User="vsc20213"
       >
       </job>
    """
    #---------------------------------------------------------------------------
    def __init__(self,job_entry):
        self.data = job_entry # OrderedDict
    #---------------------------------------------------------------------------
    def get_jobid_long(self):
        """
        :return: long jobid, includes the cluster on which it was submitted.
        :rtype: str
        """
        jobid = self.data['@DRMJID']
        return jobid
    #---------------------------------------------------------------------------
    def get_jobid(self):
        """
        :return: short jobid, just the number.
        :rtype: str
        """
        jobid = self.data['@JobID']
        return jobid
    #---------------------------------------------------------------------------
    def get_state(self):
        """
        :return: state of the job, 'R', 'C', ...
        :rtype: str, 1 character.
        """
        state = self.data['@State']
        return state
    #---------------------------------------------------------------------------
    def get_effic(self,ncores_used_on_mhost=0):
        """
        Return the efficiciency, corrected or uncorrected (*ncores_used_on_mhost*>0).

        :param int ncores_used_on_mhost: if larger than zero, represents the number of cores used on mhost and the corrected efficiency is computed (see below), otherwise the uncorrected efficiency is returned.

        According to the Adaptive Computing developers:

        "For active jobs Moab reads in the "resources_used.cput" from Torque and
        divides that by the total processor seconds dedicated to the job. This
        is the efficiency of the job. Note that this can be a delayed statistic
        coming from torque so EFFIC is just an estimate until after a job has
        finished."

        The efficiency is termed uncorrected because in the current setup of hopper
        Torque has no information from the slave nodes, and hence assumes that
        *resources_used.cput* is 0 for slave nodes. E.g. for a job running on 2 nodes,
        each with 100% efficiency, Moab will report an efficiency of 50% after 100s
        because it is computed as:

        - total processor seconds: 100s * 2 nodes * 20 cores per node = 4000
        - *resources_used.cput* on the master node = 100s * 20 cores per node = 2000
        - *resources_used.cput* on the slave  node = 100s * 20 cores per node = 2000,
          but Torque sees 0s * 20 cores per node = 0s and thus reports (2000+0)/4000 = 50%

        We can scale the efficiency to the master node as

        - 50% * number_of_cores_used_by_all_nodes / number_of_cores_used_by_master_node
          = 50% * 40/20 = 100%

        Note that the JobEntry by itself does NOT know the number_of_cores_used_by_master_node
        and thus cannot correct the effic value, unless this value is provided as ncores_used_on_mhost.

        The 'corrected' value, obviously provides only information on the master host node
        rather than on the entire job! It is our hope that if the master node is performing
        well, the slave nodes do so too.
        """
        remote.err_print('method showq.ShowqJobEntry.get_effic(self, ncores_used_on_mhost) may '
                         'report incorrect values. Use JobSample.get_effic() instead.'
                        , print_time=False
                        )
        numerator   = self.data['@StatPSUtl']
        denominator = self.data['@StatPSDed']
        try:
            value = 100*float(numerator)/float(denominator) # [%]
        except ZeroDivisionError:
            value = 100 # seems safe
        if ncores_used_on_mhost>0 and Cfg.correct_effic:
            # scale the effic value to the master host node only, i.e. "correct" it.
            value *= self.get_ncores()/ncores_used_on_mhost
        return round(value,2)
    #---------------------------------------------------------------------------
    def get_username(self):
        """
        :return: username of the user that started this job.
        """
#         :return str: username.
        value = self.data['@User']
        return value
    #---------------------------------------------------------------------------
    def get_mhost(self,short=True):
        """
        :return: the mhost node (node on wcich the job started.
        """
#         :return str: name of the master compute node.
        value = self.data['@MasterHost']
        if short:
            value = value.split('.',1)[0]
        return value
    #---------------------------------------------------------------------------
    def get_ncores(self):
        """
        :return int: total number of cores for this job .
        """
        value = int( self.data['@ReqProcs'] )
        return value
    #---------------------------------------------------------------------------

#===============================================================================
def overview_by_user(arg):
    """
    Sort key for sorting warnings by username
    """
    return arg.split(' ',1)[1]
    #---------------------------------------------------------------------------

#===============================================================================
timestamp_format = '%Y.%m.%d.%Hh%M'
#===============================================================================
def get_timestamp_now():
    """
    :return: a timestamp based on the current time.
    :rtype: str
    """
    timestamp = datetime.datetime.now().strftime(timestamp_format)
    return timestamp
    #---------------------------------------------------------------------------

#===============================================================================
class JobSample:
    """
    :param ShowqJobEntry job_entry: the job entry.
    :param Job job: the parent :class: Job object.
    :param str timestamp: the timestamp of the sample.
    """
    #---------------------------------------------------------------------------
    def __init__(self,job_entry,job,timestamp):
        assert isinstance(job_entry, ShowqJobEntry)
        assert isinstance(job, Job)
        self.showq_job_entry = job_entry
        self.parent_job      = job
        self.timestamp       = timestamp
        self.data_qstat      = Data_qstat( job.jobid )
        self.mhost_job_info  = None# NeighbouringJobInfo(self)
        self.data_sar        = None
        self.details = ''
    #---------------------------------------------------------------------------
    def check_for_issues(self):
        """
        :return: True (False) if there are (aren't) issues (all rules satisfied) for this JobSample
        """
        self.mhost_job_info  = NeighbouringJobInfo(self)
        self.warnings = []
        self.overview = ''
        self.details  = ''
        if not self.data_qstat.is_interactive_job(): #interactive jobs are ignored
            for irule,rule in enumerate(rules.the_rules):
                msg = rule.check(self)
                if msg:
                    self.warnings.append(msg)
                    self.parent_job.warning_counts[irule] += 1

        if self.warnings:
            self.parent_job.nsamples_with_warnings += 1
            return True
        else:
            return False
    #---------------------------------------------------------------------------
    def compose_overview(self):
        """
        :return: the overview text of the JobSample, composed when asked the first time.
        :rtype: str
        """
        if self.warnings:
            if self.overview:
                return self.overview
            desc = '{} {} {} {}|{}'.format( self.showq_job_entry.get_jobid()
                                          , self.showq_job_entry.get_username()
                                          , self.showq_job_entry.get_mhost()
                                          , self.get_nnodes()
                                          , self.get_ncores()
                                          )
            self.overview = '\n'+( desc.ljust(32)+self.warnings[0] ).ljust(68)+str(self.parent_job.jobscript.loaded_modules(short=True))
            spaces = '\n'+(32*' ')
            for w in self.warnings[1:]:
                w0 = w.split('\n',1)[0]
                self.overview += spaces+w0
        else:
            self.overview = ''
        return self.overview
    #---------------------------------------------------------------------------
    def compose_details(self):
        """
        :return: the details text of the JobSample, composed when asked the first time.
        :rtype: str
        """
        if self.details or not self.warnings:
            return self.details

        self.details = str(self.overview) # make a copy
        self.details += '\n\n#samples with warnings : {} / {} = {}%'.format( self.parent_job.nsamples_with_warnings
                                                                           , self.parent_job.nsamples()
                                                                           , round(100*self.parent_job.nsamples_with_warnings/self.parent_job.nsamples(),2)
                                                                           )
        for irule,count in enumerate(self.parent_job.warning_counts):
            rule = rules.the_rules[irule]
            if count>0:
                self.details +='\n  {:25}: {:5}'.format(rule.warning,count)

        self.details += '\nwalltime used/remaining: {} / {}'.format( self.data_qstat.get_walltime_used()
                                                                   , self.data_qstat.get_walltime_remaining()
                                                                   )
        mem_available = cluster_properties[current_cluster]['mem_avail_gb'](self.get_nodes())
        self.details += '\nmem [GB] used/requested/available: {} / {} / {}'.format( round(self.data_qstat.get_mem_used()     ,3)
                                                                                , round(self.data_qstat.get_mem_requested(),3)
                                                                                , mem_available
                                                                                )
        hdr = 'nodes and cores used: '
        nohdr = len(hdr)*' '
        nodes = self.data_qstat.get_exec_host().split('+')
        self.details += '\n'+hdr+nodes[0]
        for node in nodes[1:]:
            self.details += '\n'+nohdr+node

        self.details += self.mhost_job_info.to_str()

        if self.data_qstat.node_sar:
            self.details += title_line('sar -P ALL 1 1',width=100,char='-')
            if len(self.data_qstat.node_sar)>1:
                avgs = [self.get_ncores()]
                avgs.extend(6*[0])
                for data_sar in self.data_qstat.node_sar.values():
                    if hasattr(data_sar,'columns'):
                        avgs[1] += data_sar.columns['%user'  ][0]
                        avgs[2] += data_sar.columns['%nice'  ][0]
                        avgs[3] += data_sar.columns['%system'][0]
                        avgs[4] += data_sar.columns['%iowait'][0]
                        avgs[5] += data_sar.columns['%steal' ][0]
                        avgs[6] += data_sar.columns['%idle'  ][0]
                nnodes = self.get_nnodes()
                for i in range(1,7):
                    avgs[i]/=nnodes
                self.details += 'AVERAGE  '+Data_sar.line_fmt.format(*avgs)+'\n'
            for node, data_sar in self.data_qstat.node_sar.items():
                for line in data_sar.data_cores:
                    self.details += node+' '+line +'\n'
        self.details += title_line('Script',width=100,char='-')
        for line in self.parent_job.jobscript.clean:
            self.details += line+'\n'
        self.details += title_line(width=100,char='-')

        return self.details
    #---------------------------------------------------------------------------
    def walltime(self,hours=False):
        """
        :param bool hours: select the return type and format: *True->int* = #hours, *False->str* = HH:MM:SS.
        :return: the current walltime as reported by qstat, either as the number of hours *(int)*, or as HH:MM:SS *(str)*.
        :rtype: int or str.
        """
        try:
            wt = self.data_qstat.data['resources_used']['walltime']
            if hours:
                wt = '{:.2f} hrs'.format( hhmmss2s(wt)/3600 )
        except KeyError:
            if hours:
                wt = '? hrs'
            else:
                wt = '??:??:??'
        return wt
    #---------------------------------------------------------------------------
    def nodedays(self):
        """
        :return: the number of node days comsumed so far, as a formatted str.
        """
        try:
            wt = self.data_qstat.data['resources_used']['walltime']
            nd = hhmmss2s(wt)*self.get_nnodes()/(3600*24)
            nd = '{:.3f} node days'.format(nd)
        except KeyError:
            nd = '?'
        return nd
    #---------------------------------------------------------------------------
    def get_effic(self,mhost_only=Cfg.correct_effic):
        """
        Compute the efficiency from the qstat output as::

            effic = 100 * cputime_used_by_all_cores / (ncores*walltime) # percentage

        This should be close to 100% for well-performing jobs.

        :param bool mhost_only: if True, assumes that the reported efficiency is based on information from the mhost node only, an scales the reported value accordingly.
        :return: efficiency as a percentage.

        .. note:: If mhost_only is *True*, and this is a multi-node job, the value reurned
                  provides **only** information on the mhost node. If it is above the threshold,
                  it is assumed/hoped that the slave nodes are well-performing too, but there is
                  no guarantee. To be certain, the sar output must be inspected (but it is only
                  generated if the mhost is below the threshold).
                  See allso :func:`ShowqJobEntry.get_effic()`.
        """
        if not hasattr(self,'effic'):
            # we must first compute it.
            try:
                walltime                   = hhmmss2s( self.data_qstat.data['resources_used']['walltime'] )
                cputime_used_by_all_cores  = hhmmss2s( self.data_qstat.data['resources_used']['cput'] )
                ncores   = self.get_ncores()
                self.effic = 100*cputime_used_by_all_cores/(ncores*walltime)
                if mhost_only:
                    ncores_on_mhost = self.get_ncores(cn='mhost')
                    self.effic *= (ncores/ncores_on_mhost)
            except Exception as e:
                remote.err_print('JobSample.get_effic():',type(e),e)
                self.effic = 0

        return self.effic
    #---------------------------------------------------------------------------
    def get_ncores(self,cn='all'):
        """
        :param str cn: compute node name. If equal to 'mhost' the master node is taken.
        :return: number of cores in use on compute node *cn*. If *cn* is *None* the total number of cores in use
        :rtype: int
        """
        return self.data_qstat.node_cores.ncores(cnode=cn)
    #---------------------------------------------------------------------------
    def get_nnodes(self):
        """
        :return: the number of nodes used as reported by qstat.
        """
        return self.data_qstat.get_nnodes()
    #---------------------------------------------------------------------------
    def get_nodes(self):
        """
        :return: a list of the (short) node names on which the job is running.
        """
        return self.data_qstat.node_cores.nodes()
    #---------------------------------------------------------------------------
    def get_jobid(self):
        """
        :return: the jobid
        """
        return self.data_qstat.jobid
    #---------------------------------------------------------------------------
    def get_mhost(self,short=False):
        """
        Return the name of the mhost node, e.g. 'r5c2cn01.hopper.antwerpen.vsc'.
        If *short==True*, it is shortened to the part in front of the first dot: i.e.
        'r5c2cn01'.
        """
        if short:
            mhost = self.data_qstat.node_cores.mhost
        else:
            mhost = self.data_qstat.get_master_node()
        return mhost
    #---------------------------------------------------------------------------
    def get_mem(self):
        """
        :return: the maximum of memory used and requested.
        """
        memreqd = self.data_qstat.get_mem_requested()
        memused = self.data_qstat.get_mem_used()
        mem = max(memreqd,memused)
        return mem
    #---------------------------------------------------------------------------
#     def get_effic(self):
#         """
#         from showq or data_qstqt
#         """
#         if
#         if Cfg.correct_effic:
#             ncores_used_on_mhost = self.get_ncores(cn='mhost')
#         else:
#             ncores_used_on_mhost = 0
#         value = self.showq_job_entry.get_effic(ncores_used_on_mhost)
#         return value
    #---------------------------------------------------------------------------

#===============================================================================
class NeighbouringJobInfo:
    """
    Info on all jobs running on the master node of a job sample.

    :param JobSample job_sample:
    """
    def __init__(self,job_sample):
        timestamp = job_sample.timestamp
        jobid1 = job_sample.get_jobid()
        self.jobid  = [jobid1]
        self.nnodes = [job_sample.get_nnodes()]
        self.ncores = [job_sample.get_ncores()]
        self.effic  = [job_sample.get_effic ()]
        self.memory = [job_sample.get_mem   ()]

        self.mhost = job_sample.get_mhost(short=True)
        neighbouring_jobs = job_sample.parent_job.sampler.mhost_jobs[self.mhost]

        for jobid2 in neighbouring_jobs:
            if jobid2 != jobid1:
                job2 = job_sample.parent_job.sampler.jobs[jobid2]
                try:
                    job2sample = job2.get_sample(timestamp)
                except KeyError as e:
                    print(type(e),e,job2)
                    self.nnodes.append(0)
                    self.ncores.append(0)
                    self.effic .append(0)
                    self.memory.append(0)
                else:
                    self.nnodes.append(job2sample.get_nnodes())
                    self.ncores.append(job2sample.get_ncores())
                    self.effic .append(job2sample.get_effic ())
                    self.memory.append(job2sample.get_mem   ())
                self.jobid .append(jobid2)
        self.n = len(neighbouring_jobs)
        if self.n>1:
            self.jobid .append('total:')
            self.nnodes.append(1)
            self.ncores.append(sum(self.ncores))
            effic = 0
            for i in range(self.n):
                effic += self.effic[i]*self.ncores[i]
            self.effic.append( effic/self.ncores[-1] )
            self.memory.append(sum(self.memory))
    #---------------------------------------------------------------------------
    def to_str(self):
        """
        Format self as a *str*.
        """
        s = '\nother jobs on {}: '.format(self.mhost)
        if self.n == 1:
            s += 'None.'
        else:
            fmt = '\n  **{}**{:3}|{:2} {:5.1f}% {:7.3f}GB'
            i = 0
            s+= '{} (total={}).'.format(self.n-1,self.n)
            s+= fmt.format( self.jobid [i]
                          , self.nnodes[i]
                          , self.ncores[i]
                          , self.effic [i]
                          , self.memory[i]
                          )
            fmt = fmt.replace('*',' ')
            for i in range(1,self.n):
                if self.nnodes[i]!=0:
                    s += fmt.format( self.jobid [i]
                                   , self.nnodes[i]
                                   , self.ncores[i]
                                   , self.effic [i]
                                   , self.memory[i]
                                   )
                else:
                    s+= '\n    {} (no info)'.format(self.jobid[i])
        s+='\n'
        return s
    #---------------------------------------------------------------------------

#===============================================================================
def hhmmss2s(hhmmss):
    """
    Convert time duration in format HH:MM:SS to number of seconds.

    :return: duration in seconds
    :rtype: int
    """
    words = hhmmss.split(':')
    assert len(words)==3
    seconds = int(words[2]) + 60*( int(words[1]) + 60*int(words[0]) )
    return seconds
#-------------------------------------------------------------------------------

#===============================================================================
class Job:
    """
    class for storing and manipulating all data (:class:`JobSample` objects)
    related to a job.

    :param str timestamp: the timestamp of the first sample of the job.
    :param ShowqJobEntry job_entry: a job entry with the showq information of a job from a sample.
    :param Sampler sampler: (a reference to) the :class:`Sampler` object in charge.

    Occasionally, we need to examine other jobs to judge performance of a job, e.g.
    when a job is not using all resources of the node. For this reason :class:`Job`
    objects store a reference to the :class:`Sampler` object.
    """
    #---------------------------------------------------------------------------
    def __init__(self,timestamp,job_entry,sampler):
        assert isinstance(job_entry,ShowqJobEntry)
        self.jobid    = job_entry.get_jobid()
        self.username = job_entry.get_username()
        self.mhost    = job_entry.get_mhost()
        self.address  = None

        self.sampler = sampler

        self.nsamples_with_warnings = 0
        self.warning_counts = len(rules.the_rules)*[0]

        self.samples = OrderedDict() #{timestamp:JobSamnple object}
        self.first_timestamp = timestamp
        self.last_timestamp  = None
        self.jobscript       = None

        self.add_sample(job_entry,timestamp)
    #---------------------------------------------------------------------------
    def __str__(self):
        s = self.jobid    + '\n'
        s+= self.username + '\n'
        s+= self.mhost    + '\n'
        s+= str(self.samples)
        return s
    #---------------------------------------------------------------------------
    def add_sample(self,job_entry,timestamp):
        """
        Create a sample with the current *timestamp* from *job_entry*, and add it to the current Job.
        """
        self.last_timestamp = timestamp
        self.samples[timestamp] = JobSample(job_entry,self,timestamp)
    #---------------------------------------------------------------------------
    def timestamps(self):
        """
        :return: an ordered list of timestamps in this Job.
        """
        keys = list(self.samples.keys())
        return keys
    #---------------------------------------------------------------------------
#     def index(self,timestamp):
#         """
#         :return: the index of a timestamp.
#         """
#         index = self.timestamps().index(timestamp)
#         return index
    #---------------------------------------------------------------------------
    def nsamples(self):
        """
        :return: the number of samples in this Job, so far.
        """
        return len(self.samples)
    #---------------------------------------------------------------------------
    def check_for_issues(self,timestamp):
        """
        Verify wheter this job violates any of the rules for well-performing jobs.

        :return: an overview line if not all rules are satisfied, empty *str* otherwise
        """
        sample = self.samples[timestamp]
        if sample.check_for_issues():
            #there are issues
            if self.jobscript is None:
                self.jobscript = Data_jobscript(self.jobid,self.mhost)
            overview_line = sample.compose_overview()
        else:
            overview_line = ''
        return overview_line
    #---------------------------------------------------------------------------
    def overall_memory_used(self):
        """
        :return: the maximum amount of memory used by this Job, over all its samples.
        """
        mem_used = 0
        for sample in self.samples.values():
            mem_used = max(mem_used,sample.data_qstat.get_mem_used())
        return mem_used
    #---------------------------------------------------------------------------
    def get_details(self,timestamp):
        """
        :return the details text for *timestamp*.
        """
        if not timestamp in self.samples:
            timestamp = self.timestamps()[-1]
        details = self.samples[timestamp].compose_details()
        return details
    #---------------------------------------------------------------------------
    def remove_file(self):
        """
        Remove the *.pickled.gz* file corresponding to this Job.
        """
        fname = 'running/{}_{}.pickled.gz'.format(self.username,self.jobid)
        try:
            os.remove(fname)
        except:
            remote.err_print('failed to remove',fname)
    #---------------------------------------------------------------------------
    def get_sample(self,timestamp='last'):
        """
        Return the sample corresponding to *timestamp*, or, if it is *None*, the last sample.
        """
        if timestamp=='last':
            sample = od_last(self.samples)[1]
        else:
            sample = self.samples[timestamp]
        return sample
    #---------------------------------------------------------------------------
    def get_nnodes(self,timestamp='last'):
        """
        :return: the numer of nodes as reported by the sample at *timestap*
        """
        sample = self.get_sample(timestamp)
        return sample.get_nnodes()
    #---------------------------------------------------------------------------
    def get_ncores(self,timestamp='last'):
        """
        :return: the numer of nodes as reported by the sample at *timestap*
        """
        sample = self.get_sample(timestamp)
        return sample.get_ncores()
    #---------------------------------------------------------------------------
    def get_mem(self,timestamp='last'):
        """
        :return: the maximum of memory used and requested for the given timestamp.
        """
        sample = self.get_sample(timestamp)
        return sample.get_mem()
    #---------------------------------------------------------------------------
    def pickle(self,prefix,only_if_warnings=True,verbose=False,compressed=True):
        """
        Pickle this job and compress it using gzip.

        :param str prefix: the receiving directory.
        :param bool only_if_warnings: do only pickle if the job has warnings.
        :param bool verbose: if *True*, print the destination file.
        :param bool compressed: compress after pickling.
        """
        if (only_if_warnings and self.nsamples_with_warnings) \
        or (not only_if_warnings):
            #pickle
            if 'running' in prefix:
                fname = '{}_{}.pickled'   .format(self.username,self.jobid)
            else:
                fname = '{}_{}_{}.pickled'.format(self.username,self.jobid,self.timestamps()[-1])

            if compressed:
                fpath = os.path.join(prefix,fname+'.gz')
                fo = gzip.open(fpath,'wb')
            else:
                fo =      open(fpath,'wb')
            # remove the "upward" object references in the data tree
            # otherwise they waste a lot of disk space
            sampler = self.sampler
            self.sampler = None
            # job_sample.parent_job
            for job_sample in self.samples.values():
                job_sample.parent_job = None
            # pickle this job
            pickle.dump(self,fo)
            # finally restore the upward references
            self.sampler = sampler
            # job_sample.parent_job
            for job_sample in self.samples.values():
                job_sample.parent_job = self

            if verbose:
                print(' (pickled {})'.format(fpath))
            fo.close()
    #---------------------------------------------------------------------------

#===============================================================================
def unpickle(fpath,sampler=None,verbose=False):
    """
    Counterpart of Job.pickle()

    :param str fpath: path to pickled file. If ending on '.pickled.gz', unzips before unpickling. If ending on '.pickled', unpickle without unzipping. Otherwise try both in that order.
    :param Sampler sampler: :class:`Sampler` object or :class:`None`.
    :param bool verbose: print the filename of the unpickled file.
    :return: a Job object or :class:`None` if the file does not exist.
    """
    if fpath.endswith('.pickled.gz'):
        try:
            with gzip.open(fpath,'rb') as fo:
                job = pickle.load(fo)
        except:
            job = None

    elif fpath.endswith('.pickled'):
        try:
            with open(fpath,'rb') as fo:
                job = pickle.load(fo)
        except:
            job = None
    else:
        _fpath = fpath+'.pickled.gz'
        if os.path.exists(_fpath):
            return unpickle(_fpath, sampler=sampler, verbose=verbose)
        else:
            _fpath = fpath+'.pickled'
            if os.path.exists(_fpath):
                return unpickle(_fpath, sampler=sampler, verbose=verbose)
            else:
                job = None
    if job is None:
        if verbose:
            print(' (not found {})\n'.format(fpath))
    else:
        if verbose:
            print(' (unpickled {})\n'.format(fpath))
        # Set the "upward" object references in the data tree
        # otherwise they refer to their version at the moment of pickling
        job.sampler = sampler
        # job_sample.parent_job
        for job_sample in job.samples.values():
            job_sample.parent_job = job
    return job
#===============================================================================
class Sampler:
    """
    Class that does the sampling, either through :func:`sample` or :func:`fetch_offline_samples`.
    The frequency of sampling is determined by the caller.

    :param int interval: number of seconds between successive samples.
    :param qMainWindow: If *None* prints a progress bar to the terminal during sampling, otherwise uses a Qt4:QProgressDialog.
    """
    #---------------------------------------------------------------------------
    def __init__(self,interval=None,qMainWindow=None):
        if interval is None:
            self.sampling_interval = Cfg.sampling_interval
        else:
            self.sampling_interval = interval
        self.qMainWindow = qMainWindow

        self.overviews = OrderedDict()      # {timestamp:job_overview}
        self.jobs    = {}                   # {jobid    :Job object  }
        self.timestamps = []                # [datetime.strftime(timestamp_format)]
        self.timestamp_jobs = OrderedDict() # {timestamp:[jobids]}
        self.jobids_running_previous = []
    #---------------------------------------------------------------------------
    def sample(self,verbose=False,show_progress=False):
        """
        Sample the running jobs online (locally).
        """
        self.data_showq = remote.run("showq -r -p hopper --xml",post_processor=remote.xml_to_odict)
        self.total_nodes_in_use = self.get_total_nodes_in_use()
        # get the job entries
        try:
            job_entries = self.data_showq['Data']['queue']['job']
        except:
            remote.err_print('No jobs running.')
            exit(0)
        # remove jobs
        #  . which have no mhost set
        #  . which have jobid like '390326[1]' (=job array jobs)
        job_entries_filtered = []
        for job_entry in job_entries:
            converted = ShowqJobEntry(job_entry)

            # ignore jobs with unknow mhost
            try:
                converted.get_mhost()
            except KeyError:
                print('ignoring',converted.get_jobid_long(), '(mhost unknown)')
                continue

            # ignore jobs with jobid containing '[n]'
            jobid = converted.get_jobid()
            if '[' in jobid:
                print('ignoring',job_entry.get_jobid_long(), '(worker job)')
                continue
            job_entries_filtered.append(converted)
        job_entries = job_entries_filtered

        self.n_entries   = len(job_entries)

        if self.qMainWindow:
            from PyQt4.QtGui import QProgressDialog,QApplication
            dlg = QProgressDialog('','',0, self.n_entries,self.qMainWindow)
            hdr = 'Sampling #{} : {} {}/{}'
        else:
            if show_progress:
                from progress import printProgress
                hdr = 'sampling showq #{}'.format(len(self.timestamp_jobs)+1)

        # create
        #   . a dict { mhost : [jobid] } with all the jobs running on node mhost
        #   . a list wit all uncompleted jobids
        # the latter is compared to the jobid list of the previous sample to find
        # out which jobs are finished.
        self.mhost_jobs = OrderedDict()
        self.jobids_running = []
        for job_entry in job_entries:
            mhost = job_entry.get_mhost()
            jobid = job_entry.get_jobid()
            od_add_list_item(self.mhost_jobs,mhost,jobid)
            self.jobids_running.append(jobid)
            try:
                self.jobids_running_previous.remove(jobid)
            except ValueError:
                pass
            #   when this loop has completed, self.jobids_running_previous
            #   contains only jobides of finished jobs.
        jobids_finished = self.jobids_running_previous
        self.jobids_running_previous = self.jobids_running # prepare for next sample() call
        # pickle finished jobs (if they had issues) and remove them from self.jobs
        os.makedirs('completed', exist_ok=True)
        for jobid in jobids_finished:
            try:
                job = self.jobs.pop(jobid)
            except KeyError:
                continue
            job.pickle('completed/',verbose=True)
            if Cfg.offline:
                job.remove_file()
        timestamp = get_timestamp_now()
        if Cfg.offline:
            os.makedirs ('running',exist_ok=True)
            if os.path.exists('running/timestamp'):
                os.remove('running/timestamp')
            #   if ths file is absent ojm is sampling.
            print(title_line(timestamp, char='=', width=100, above=True, below=True),end='')

        # loop over the running jobs (job_entries)
        #pass 1 create jobs and job samples
        for i_entry,job_entry in enumerate(job_entries):
            if job_entry.get_state() != 'Running':
                continue # we only analyze running jobs
            jobid    = job_entry.get_jobid()
            #username = job_entry.get_username()
            od_add_list_item(self.timestamp_jobs,timestamp,jobid)

            if self.qMainWindow:
                progress_message = hdr.format(len(self.timestamp_jobs),jobid,i_entry,self.n_entries)
                dlg.setLabelText(progress_message)
                dlg.setValue(i_entry)
                QApplication.processEvents()
            else:
                if show_progress:
                    printProgress(i_entry, self.n_entries, prefix=hdr, suffix='jobid='+jobid, decimals=-1)

            job = self.jobs.get(jobid,None)
            if job is None:
                # this job is encountered for the first time
                job = Job(timestamp,job_entry,self)
                self.jobs[jobid] = job
            else:
                job.add_sample(job_entry,timestamp)
                if job.sampler is None:
                    remote.err_print('### strange')
                    job.sampler = self

        if self.qMainWindow:
            # terminate QProgressDialog
            dlg.setValue(self.n_entries)
            QApplication.processEvents()
            # start new QProgressDialog
            dlg = QProgressDialog('','',0, self.n_entries,self.qMainWindow)
            hdr = 'Checking rules #{} : {} {}/{}'
        else:
            if show_progress:
                # terminate printProgress
                printProgress(self.n_entries, self.n_entries, prefix=hdr, suffix='', decimals=-1)
                # start new printProgress
                hdr = 'Checking rules #{}'.format(len(self.timestamp_jobs)+1)

        #pass 2 add NeighbouringJobInfo and check the rules
        overview = [] # one warning per job with issues, jobs without issues are skipped
        i_entry = 0
        for jobid,job in self.jobs.items():
            #progress
            if self.qMainWindow:
                progress_message = hdr.format(len(self.timestamp_jobs),jobid,i_entry,self.n_entries)
                dlg.setLabelText(progress_message)
                dlg.setValue(i_entry)
                QApplication.processEvents()
            else:
                if show_progress:
                    printProgress(i_entry, self.n_entries, prefix=hdr, suffix='jobid='+jobid, decimals=-1)
            i_entry += 1
            #the real work
            overview_line = job.check_for_issues(timestamp)
            if overview_line:
                overview.append(overview_line)
                if verbose:
                    print('\n'+timestamp+'\n')
                    print(job.get_details(timestamp))
                if Cfg.offline:
                    job.pickle('running', verbose=verbose)

        if self.qMainWindow:
            # terminate QProgressDialog
            dlg.setValue(self.n_entries)
            QApplication.processEvents()
        else:
            if show_progress:
                # terminate printProgress
                printProgress(self.n_entries, self.n_entries, prefix=hdr, suffix='', decimals=-1)
            for line in overview:
                print(line,end='')
            print('\nWell performing jobs: {}/{}'.format(self.n_entries-len(overview),self.n_entries))
        print(self.total_nodes_in_use)

        if Cfg.offline:
            # notify that sampling has finished..
            with open('running/timestamp','w') as f:
                f.write(timestamp)

        self.overviews[timestamp] = self.overview_list2str(overview)

        self.timestamps.append(timestamp)
        #    this must be the last statement because the gui otherwise sees a timestamp which is not ready.
        return timestamp
    #---------------------------------------------------------------------------
    def get_total_nodes_in_use(self):
        """
        :return: a str describing the fraction of nodes in use.
        """
        if hasattr(self,'data_showq'):
            s = 'nodes in use: {}/{}'.format( self.data_showq['Data']['cluster']['@LocalActiveNodes']
                                            , self.data_showq['Data']['cluster']['@LocalConfigNodes'] )
            self.total_nodes_in_use = s
            return s
        else:
            if hasattr(self,'total_nodes_in_use'):
                return self.total_nodes_in_use
            else:
                return ''
    #---------------------------------------------------------------------------
    def get_remote_timestamp(self):
        """
        :return: the last sample's timestamp from the offline job monitor. If ojm.py is in the process of sampling an empty string is returned.
        """
        try:
            lines = remote.run('cd data/jobmonitor/running/; cat timestamp',post_processor=remote.list_of_lines)
            return lines[0]
        except:
            return ''
    #---------------------------------------------------------------------------
    def fetch_offline_samples(self):
        """
        Sample the running jobs from the offline job monitor. The remote directory '~/data/jobmonitor/running'
        examined to see if there are new samples available. These are copied to the local directory ./offline/running
        """
        shutil.rmtree('offline/running')
        os.makedirs('offline/running'  ,exist_ok=True)
        #os.makedirs('offline/completed',exist_ok=True)
        timestamp = self.get_remote_timestamp()
        while not timestamp:
            sleep(60)
            timestamp = self.get_remote_timestamp()
        if self.timestamps:
            if timestamp==self.timestamps[-1]:
                return # this timestamp is already in the samples
        self.timestamps.append(timestamp)
        filenames = remote.glob('*.pickled.gz','data/jobmonitor/running/')
        self.n_entries = 0
        for filename in filenames:
            if not filename: # empty line
                continue
            lfname =         'offline/running/'+filename
            rfname = 'data/jobmonitor/running/'+filename
            print('copying '+rfname,'to',lfname,end='')
            try:
                remote.copy_remote_to_local(lfname,rfname)
                print(' - copied')
            except:
                print(' - failed')
                continue
            job = unpickle('offline/running/'+filename,sampler=self)
            self.add_offline_job(job)
            self.n_entries += 1
        self.overviews[timestamp] = self.overview_list2str(self.overviews[timestamp])
    #---------------------------------------------------------------------------
#     def timestamp(self,i=-1):
#         return self.timestamps[i]
    #---------------------------------------------------------------------------
    def overview_list2str(self,overview_list,key=overview_by_user,reverse=True):
        """
        Sort the *overview_list* according to *key* and *reverse* and convert to plain text.
        """
        overview_list.sort(key=key,reverse=reverse)
        n_jobs = self.n_entries
        n_warn = len(overview_list)
        text = 'Jobs running well: {}/{}, efficiency threshold = {}%'.format(n_jobs-n_warn,n_jobs,Cfg.effic_threshold)
        s = self.get_total_nodes_in_use()
        if s:
            text+= ', '+s
        text+= ''.join(overview_list)
        return text
    #---------------------------------------------------------------------------
    def nsamples(self):
        """
        :return: the number of samples.
        """
        return len(self.timestamps)
    #---------------------------------------------------------------------------
    def add_offline_job(self,job):
        """
        Add an offline monitored *job* to the sampler.
        """
        self.jobs[job.jobid] = job
        for timestamp,job_sample in job.samples.items():
            od_add_list_item(self.timestamp_jobs,timestamp,job.jobid)
            overview_line = job_sample.compose_overview()
            if not timestamp in self.overviews:
                self.overviews[timestamp] = [overview_line]
            else:
                overview = self.overviews[timestamp]
                if isinstance(overview,str):
                    lines = overview.split('\n')
                    self.overviews[timestamp] = []
                    overview = self.overviews[timestamp]
                    for line in lines:
                        if not line.endswith('\n'):
                            line += '\n'
                        overview.append(line)
                overview.append(overview_line)
        self.total_nodes_in_use = job.sampler.get_total_nodes_in_use()
    #---------------------------------------------------------------------------
    def when_done_adding_offline_jobs(self):
        """
        Sort offline monitored jobs.
        """
        self.timestamps.sort()
        for jobid_list in self.timestamp_jobs.values():
            jobid_list.sort()
    #---------------------------------------------------------------------------

################################################################################
# test code below
################################################################################
if __name__=="__main__":
    remote.connect_to_login_node()

    sampler = Sampler()
    timestamp = sampler.sample()

    current_jobids = sampler.timestamp_jobs[timestamp]
    for jobid in current_jobids:
        job_sample = sampler.jobs[jobid].samples[timestamp]
        print( job_sample.compose_details(),end='')

    print('\n--finished--')


# -*- coding:utf-8 -*-
class Solution(object):
    def containsNearbyDuplicate(self, nums, k):
        if len(nums) <= 1:
            return False
        dic = {}
        for key, value in enumerate(nums):
            if value in dic and key - dic[value] <= k:
                return True
            dic[value] = key
        return False

'''
# --- TLE ---
class Solution(object):
    def containsNearbyDuplicate(self, nums, k):
        if len(nums) <= 1:
            return False
        for i in range(len(nums)):
            while k >= 0:
                if i+k <= len(nums)-1:
                    if nums[i] == nums[i+k]:
                        return True
                else:
                    k -= 1
        return False
'''


import openturns as ot
from openturns.viewer import View

ot.RandomGenerator.SetSeed(0)

size = 100
normal = ot.Normal(1)
sample = normal.getSample(size)

henryPlot = ot.VisualTest.DrawHenryLine(sample)

View(henryPlot, figure_kw={"figsize": (4.5, 4.5)})


# -*- coding: utf-8 -*-

import rdflib
import subprocess
import sys
import codecs

sys.stdout = codecs.getwriter("utf-8")(sys.stdout)

# On créé un graph RDF et on charge le contenu de la base de données Zotero
# dans ce graph
g = rdflib.Graph()
g.load('Zotero-AK.rdf')

# On déclare les espace de noms utilisés dans Zotero
ns = {
    'bib': rdflib.Namespace("http://purl.org/net/biblio#"),
    'z': rdflib.Namespace("http://www.zotero.org/namespaces/export#"),
    'dc': rdflib.namespace.DC,
    'foaf': rdflib.namespace.FOAF
}

qs = """SELECT ?doc ?title
WHERE {
    ?doc a bib:Document .
    ?doc z:itemType "blogPost" .
    ?doc dc:title ?title .
}
"""

results = g.query(qs, initNs=ns)

for res in results:
    sys.stdout.write(u"%s\t%s\n" % (res.doc[7:], res.title))
    #cmd = u"webtocon %s screenshots/%s.png 1280 1024" % (res.doc, res.title)
    #p1 = subprocess.Popen(cmd.split(" "), stdout=subprocess.PIPE, stdin=subprocess.PIPE)
    #(stdout_data, stderr_data) = p1.communicate()


import block, blockchain
from commands.helper import today

def pushBlock(block, BLOCKCHAIN):
    """Update Blockchain"""
    if BLOCKCHAIN.isChainValid() == False:
        print('The current Blockchain is not valid, performing rollback.')
        BLOCKCHAIN = blockchain.Blockchain()

    BLOCKCHAIN.addBlock(block)
    if BLOCKCHAIN.isChainValid():
        BLOCKCHAIN.storeChain()

def pushItem(id, name, item, BLOCKCHAIN):
    """Generate new Block"""
    item_block = block.Block(
        user = id,
        name = name,
        timestamp = today(),
        description = item,
        data = 0
    )

    pushBlock(item_block, BLOCKCHAIN)

def pushWish(id, name, BLOCKCHAIN):

    """Generate new Block"""
    wish_block = block.Block(
        user = id,
        name = name,
        timestamp = today(),
        description = 'Wish',
        data = 0
    )

    """Update Blockchain"""
    pushBlock(wish_block, BLOCKCHAIN)

    return


# Sumo-database
# Author: Vu Tran
""" This is a GUI database management application to store Sumo wrestlers information """
from tkinter import *
from GUI import GUI

# Main function to run the application
def main():
    window = Tk()
    application = GUI(window)
    window.mainloop()

if __name__ == "__main__":
    main()


class Solution(object):
    def maxDiff(self, num):
        """
        :type num: int
        :rtype: int
        """
        digits=str(num)
        maxNum=0
        minNum=999999999
        for i in range(10):
            tmp = digits
            for j in range(10):
                newstr=tmp.replace(str(i),str(j))
                if int(newstr) > maxNum and int(newstr) > 0 and len(str(int(newstr))) == len(digits):
                    maxNum=int(newstr)
                if int(newstr) < minNum and int(newstr) > 0 and len(str(int(newstr))) == len(digits):
                    minNum=int(newstr)
        return maxNum-minNum

sln=Solution()
assert 888==sln.maxDiff(num = 555)
assert 8==sln.maxDiff(num = 9)
assert 820000==sln.maxDiff(num = 123456)
assert 80000==sln.maxDiff(num = 10000)
assert 8700==sln.maxDiff(num = 9288)

5385. Max Difference You Can Get From Changing an Integer.py


import itertools
from operator import truediv
import subprocess
import time
import timeit
from unittest.mock import patch
from Cython.Compiler.TreeFragment import *
from Cython.Compiler.Visitor import *
from Cython.Compiler.Main import *
from Cython.CodeWriter import *
from Cython.Compiler.AnalysedTreeTransforms import *
from Cython.Compiler.ParseTreeTransforms import *
from Cython.Compiler.Symtab import *
from algorithm.algorithm import Algorithm
import utilities.cython_tree as cy
from edit import TypeInsertion
from patch import Patch
from examples import fib

class ExhaustiveSearch(Algorithm):
    def __init__(self, program):
        self.program = program

    def run(self):
        run_start = time.time()
        normal = NormalizeTree(None)
        types = ['short', 'long']
        ast = cy.file_to_ast(self.program.file_path)
        normal.visit(ast)
        modifications = len(self.program.modification_points)
        points = [x for x in range(modifications)]

        patches = []

        combinations = [p for p in itertools.product(
            types, repeat=modifications)]

        for c in combinations:
            zipped = zip(c, points)
            patches.append(list(zipped))
        best_patch = Patch(self.program)

        print(patches)

        start = timeit.default_timer()
        expected = fib.fib(75)
        end = timeit.default_timer()
        fastest = end-start
        fast_index = 0
        print(fastest)

        file = 'output/temp.pyx'
        result_list = {"success": 0, "compilation_fail": 0,
                       "bug_found": 0, "wrong_value": 0}

        for i in range(len(patches)):
            print(i)
            new_patch = best_patch.clone()
            for edit in patches[i]:
                new_patch.add(TypeInsertion.create(
                    self.program, edit[1], edit[0]))
            result = self.program.evaluate_patch(new_patch)
            print(result)

            result_list[result["status"]] += 1

            if result["status"] == "success":
                value = float(result["value"])
                if value < fastest:
                    fastest = value
                    fast_index = i
                    best_patch = new_patch

        print(fast_index)
        print(fastest)
        print(result_list)
        self.program.write_result(best_patch)

        run_end = time.time()
        print(run_end-run_start)

        return


def solve():
    acc = 0
    for i in range(1,1001):
        acc += numberWord(i)

    print(acc)

def numberWord(x):
    if      x < 20:
        return lessThanTwenty(x)
    elif    len(str(x)) == 2:
        return twoDigit(x)
    elif    len(str(x)) == 3:
        return threeDigit(x)
    else:
        return 11 #onethousand

def lessThanTwenty(x):
    if x < 10:  return ones(str(x))
    else:
        return tenToTwenty(str(x))

def twoDigit(x):
    digitSplit = list(str(x))

    if digitSplit[1] == '0':
        return tens(digitSplit[0])
    else:
        return tens(digitSplit[0]) + ones(digitSplit[1])

def threeDigit(x):
    digitSplit = list(str(x))

    if digitSplit[1] == '0' and digitSplit[2] == '0':
        return ones(digitSplit[0]) + 7
    elif int(digitSplit[1]) < 2:
        return ones(digitSplit[0]) + 7 + 3 + lessThanTwenty(int(digitSplit[1] + (digitSplit[2])))
    else:
        return ones(digitSplit[0]) + 7 + 3 + tens(digitSplit[1]) + ones(digitSplit[2])

def ones(x):
    if      x == '0': return 0
    elif    x == '1': return 3
    elif    x == '2': return 3
    elif    x == '3': return 5
    elif    x == '4': return 4
    elif    x == '5': return 4
    elif    x == '6': return 3
    elif    x == '7': return 5
    elif    x == '8': return 5
    else            : return 4

def tenToTwenty(x):
    if      x == '10': return 3
    elif    x == '11': return 6
    elif    x == '12': return 6
    elif    x == '13': return 8
    elif    x == '14': return 8
    elif    x == '15': return 7
    elif    x == '16': return 7
    elif    x == '17': return 9
    elif    x == '18': return 8
    else             : return 8

def tens(x):
    if        x == '2': return 6
    elif      x == '3': return 6
    elif      x == '4': return 5
    elif      x == '5': return 5
    elif      x == '6': return 5
    elif      x == '7': return 7
    elif      x == '8': return 6
    else              : return 6

solve()

#21124


#MAP and REDUCE are very effective ways to handle data structure. Here are
#Some examples.

#   1. I have an array of Temperature in Degree F. I want to convert them to
#   Degree C. Use map to convert the temperature unit.

DegreeF = [-40, 0,0,20,30,40, 50, 60, 70, 80, 90, 100, 110, 120, 320]

def convert_F_to_C(T):
    '''
    Conversion formula:
    (C-0)/100-0 = (F-32)/(212-32)
    --> C= (F-32)*100/180
    --> C = (F-32)*5/9
    '''
    return round((T-32)*5/9,2) #round in two decimal places

#Note in python 3, you need to pass the map in list to convert it to list
#Note, even though convert_F_to_C method has argument T, in map we pass the
#argument as a parameter of map method.
#Here: map(convert_F_to_C,DegreeF)==> map(function, argument of function)
DegreeC= list(map(convert_F_to_C,DegreeF))

for i in range(len(DegreeF)):
    print (DegreeF[i],"F =  ", DegreeC[i],"C")

# 2. Similar example: given I have an array x, create another array that is
# x^3-2x+2

import random
arr = [random.randint(0,10) for _ in range (10)]
print ("Original array: \n",arr)

new_arr = list(map(lambda x: x**3 -2*x +2, arr))

#Here lambda expression is short form of function
"""
def func(x):
    return x**3 -2*x +2
"""

print("Original array mapped to a function: \n",new_arr)

# 3. Find sum of all elements of an array using reduce
arr = [random.randint(-3,3) for _ in range (10)]
print ("Original array: \n",arr)

from functools import reduce #In general you need to import this library to use reduce

sum_of_all = reduce(lambda x, y :x+y, arr)
print ("sum_of_all: ", sum_of_all)

#Equivalent function for lambda expression:
def func(x,y):
    return x+y

sum_of_all_1 = reduce(func, arr)

print ("sum_of_all_1: ", sum_of_all_1)

# 4. Use reduce to find max value in an array

def myMax(x,y):
    if x>y:
        return x
    else:
        return y
maxValue  = reduce (myMax,arr)

print ("Original array: \n",arr)
print ("Max value of array: ",maxValue)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = 'Daniel Zhang (張道博)'
__copyright__ = 'Copyright (c) 2014, University of Hawaii Smart Energy Project'
__license__ = 'https://raw.github' \
              '.com/Hawaii-Smart-Energy-Project/Maui-Smart-Grid/master/BSD' \
              '-LICENSE.txt'

import unittest
from sek.logger import SEKLogger
from msg_types import MSGAggregationTypes

class MSGTypesTester(unittest.TestCase):
    """
    Unit tests for MSG Aggregation Types.
    """

    def setUp(self):
        self.logger = SEKLogger(__name__, 'DEBUG')

    def test_aggregation_types(self):
        self.assertTrue(MSGAggregationTypes.weather in MSGAggregationTypes)
        self.assertTrue(MSGAggregationTypes.egauge in MSGAggregationTypes)
        self.assertTrue(MSGAggregationTypes.circuit in MSGAggregationTypes)
        self.assertTrue(MSGAggregationTypes.irradiance in MSGAggregationTypes)

    def tearDown(self):
        pass

if __name__ == '__main__':
    RUN_SELECTED_TESTS = True

    if RUN_SELECTED_TESTS:

        selected_tests = ['test_aggregation_types']

        mySuite = unittest.TestSuite()
        for t in selected_tests:
            mySuite.addTest(MSGTypesTester(t))
        unittest.TextTestRunner().run(mySuite)
    else:
        unittest.main()


#
# This project describes the detection of varroas using opencv's
# Simple Blob Detector with some arbitrary but seemingly working parameters
#
# version originale de Vincent SAHLER : https://gist.github.com/vsahler/9686d9b12d63bb1f1481aa957bd1933e
#
# documentation : https://learnopencv.com/blob-detection-using-opencv-python-c/
#               : https://docs.opencv.org/3.4/d8/da7/structcv_1_1SimpleBlobDetector_1_1Params.html
#
# rapport entre longueur et largeur de l'ellipse : 1.6 /1.1  surface 1.3816

import cv2
import numpy as np
import sys
import time
import imutils
import csv;

#names=["./Michel_0_3_1080p.jpg","./Michel_0_4_1080p.jpg","./Image_4_246.jpg","./Image_4_89.jpg","./CM_Cal_22V_A4.jpg","./CM_Cal_50V_A4.jpg"]
#detect_names=["./detect_Michel_0_3_1080p.jpg","./detect_Michel_0_4_1080p.jpg","./detect_Image_4_246.jpg","./detect_Image_4_89.jpg","./detect_CM_Cal_22V_A4.jpg","./detect_CM_Cal_50V_A4.jpg"]
#targets=[71,335,246,89,22,50]
#kii =6

names=["./Michel_0_6.jpg","./Michel_0_7.jpg","./Michel_0_8.jpg"]
detect_names=["./detect_Michel_0_6.jpg","./detect_Michel_0_7.jpg","./detect_Michel_0_8.jpg"]
targets=[4,5,5]
kii =3

#names=["./Michel_0_3_1080p.jpg"]
#detect_names=["./detect_Michel_0_3_1080p.jpg"]
#targets=[71]
#kii =1

# dataset71 = cv2.imread("./Michel_0_3_1080p.jpg")  # 74  /  71 comptés   4% prés  propre
# dataset335 = cv2.imread("./Michel_0_4_1080p.jpg") # 415 / 335 comptés  23% prés  assez sale
# dataset246 = cv2.imread("./Image_4_246.jpg")      # 313 / 246 comptés  27% prés  assez sale
# dataset89 = cv2.imread("./Image_4_89.jpg")        # 100 /  89 comptés  12% prés  assez sale
# datasetCM22 = cv2.imread("./CM_Cal_22V_A4.jpg")   #  21 /  22 comptés   5% prés  propre
# datasetCM30 = cv2.imread("./CM_Cal_30V_A4.jpg")   #   1 /  30 comptés  99% prés  propre !!!
# datasetCM50 = cv2.imread("./CM_Cal_50V_A4.jpg")   #  40 /  50 comptés   5% prés  propre
# datasetCM100 = cv2.imread("./CM_Cal_100V_A4.jpg") #   1 / 100 comptés  99% prés  propre mais flou !!!

# Thresholds
# params.minThreshold = 15   #  original 10
# params.maxThreshold = 180  #  original 200

# Filter by filterByColor  => blobColor = 0 sombre / blobColor = 255 clair
# eviter les taches de pollen
# params.filterByColor  = True
# params.blobColor  = 10

# Filter by Area
# params.filterByArea = True
# params.minArea = 70   #  23 mais avec une marge  50
# params.maxArea = 110  # 120 mais avec marge     150

# Filter by Circularity : un cercle a une circularité de 1, la circularité d'un carré est de 0,785,
# params.filterByCircularity = True
# params.minCircularity = 0.1

# Filter by Convexity : Aire du Blob / Aire de son enveloppe convexe  1.38/2 =0.69
# params.filterByConvexity = True
# params.minConvexity = 0.69

# Filter by Inertia # un cercle, cette valeur est 1, par une ellipse , il est compris entre 0 et 1, et pour une ligne est de 0
# params.filterByInertia = True
# params.minInertiaRatio = 0.52

def analyse(ki,parameters):
	name=names[ki]
	target=targets[ki]
	workingImage = cv2.imread(name)

	# traitement de l'image en nuance de gris
	# workingImage = cv2.cvtColor(workingImage1, cv2.COLOR_BGR2GRAY)
	# fait un flougaussien
	# workingImage = cv2.bilateralFilter(workingImage1, 18, 90,100)       # initialement à 6 , 157 , 157

	a,b,c,d,e,f,g,h=parameters
	params = cv2.SimpleBlobDetector_Params()
	params.minThreshold = a          #  = 15   #  original 10
	params.maxThreshold = b          #  = 180  #  original 200
	params.filterByColor  = True
	params.blobColor  = c            # blobColor = 0 sombre / blobColor = 255 clair
	params.filterByArea = True
	params.minArea = d               #  23 mais avec une marge  50
	params.maxArea = e               # 120 mais avec marge     150
	params.filterByCircularity = True
	params.minCircularity = f        # params.minCircularity = 0.1
	params.filterByConvexity = True
	params.minConvexity = g          # params.minConvexity = 0.69
	params.filterByInertia = True
	params.minInertiaRatio = h       # params.minInertiaRatio = 0.52

	detector = cv2.SimpleBlobDetector_create(params)

	g = cv2.cvtColor(workingImage, cv2.COLOR_BGR2GRAY)

	keyPoints = detector.detect(g)

	im_with_keypoints = cv2.drawKeypoints(workingImage, keyPoints, np.array([]), (0, 0, 255), # vert
									  cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)

	nb_varroas=len(keyPoints)
	return nb_varroas,abs(nb_varroas-target),im_with_keypoints

minv=1000000
mina=[]

A=range(108,109,1)            #Thresholds  = 15   #  original 10  Seuil bas
B=range(155,156,1)            #Thresholds  = 180  #  original 200 Seuil haut
C=range(0,1,1)                #blobColor                        Color : 0 sombre  et 255 clair
D=range(109,110,1)            #Area  23 mais avec une marge  50    Aire mini
E=range(129,130,1)            #Area 120 mais avec marge     150    Aire maxi
F=np.arange(0.66,0.67,0.1)      #Circularity  = (0.8,1,0.1)    Circularity : un cercle a une circularité de 1, la circularité d'un carré est de 0,785,
G=np.arange(0.72,0.73,0.1)      #Convexity    = (0.5,0.7,0.1)    Convexity : Aire du Blob / Aire de son enveloppe convexe  1.38/2 =0.69
H=np.arange(0.44,0.45,0.1)      #InertiaRatio = (0.4,0.6,0.1)    Inertia : pour un cercle : 1, par une ellipse compris entre 0 et 1, pour une ligne : 0
nb_tests = len(A)*len(B)*len(C)*len(D)*len(E)*len(F)*len(G)*len(H)
print(nb_tests,'tests ', time.ctime(), 'duree estimée : ', int(0.5 + nb_tests * 0.1386 / 60) , 'minutes par image sur mon i5 8G')
# sur mon i5 il faut 0.1386 secondes par itération

KI=range(0,kii,1)  # domaine des photos
for ki in KI:    # indice de la photo
	print('nom de la photo : ', names[ki], ' nb de varroas comptés manuellement :', targets[ki])
	k1=0
	for h in H:
		for g in G:
			for f in F:
				for c in C:
					for b in B:
						for a in A:
							for e in E:
								for d in D:
									if a<b and d<e:

										# print(a,b,d,e)
										parameters=(a,b,c,d,e,f,g,h)

										_,diff,im_with_keypoints=analyse(ki,parameters)

										if diff<minv:
											minv=diff
											mina=(a,b,c,d,e,f,g,h)

										if (k1%100)==0:
											print('.',end='')
											sys.stdout.flush()
										k1+=1

	a,b,c,d,e,f,g,h=mina
	parameters=(a,b,c,d,e,f,g,h)
	nb_varroas,diff,im_with_keypoints=analyse(ki,parameters)
	print(f'A:{a} B:{b} C:{c} D:{d} E:{e} F:{f} G:{g} H:{h} ', nb_varroas, 'varroas détectés  pourcentage : ', int(100*(nb_varroas-targets[ki])/targets[ki]), ' %', time.ctime())

	params = cv2.SimpleBlobDetector_Params()
	# Thresholds
	params.minThreshold = a   #  original 10
	params.maxThreshold = b  #  original 200

	# Filter by filterByColor  => blobColor = 0 sombre / blobColor = 255 clair
	# eviter les taches de pollen
	params.filterByColor  = True
	params.blobColor  = c

	# Filter by Area
	params.filterByArea = True
	params.minArea = d   #  23 mais avec une marge  50
	params.maxArea = e  # 120 mais avec marge     150

	# Filter by Circularity : un cercle a une circularité de 1, la circularité d'un carré est de 0,785,
	params.filterByCircularity = True
	params.minCircularity = f

	# Filter by Convexity : Aire du Blob / Aire de son enveloppe convexe  1.38/2 =0.69
	params.filterByConvexity = True
	params.minConvexity = g

	# Filter by Inertia : un cercle, cette valeur est 1, par une ellipse , il est compris entre 0 et 1, et pour une ligne est de 0
	params.filterByInertia = True
	params.minInertiaRatio = h

	workingImage = cv2.imread(names[ki])
	detector = cv2.SimpleBlobDetector_create(params)

	g1 = cv2.cvtColor(workingImage, cv2.COLOR_BGR2GRAY)

	keyPoints = detector.detect(g1)

	im_with_keypoints = cv2.drawKeypoints(workingImage, keyPoints, np.array([]), (4, 178, 16),cv2.DRAW_MATCHES_FLAGS_NOT_DRAW_SINGLE_POINTS)
	#									  cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)

	print(f'Il y a : {len(keyPoints)} varroas détectés ')
	print(f'A:{a} B:{b} C:{c} D:{d} E:{e} F:{f} G:{g} H:{h} ', nb_varroas, 'varroas détectés  pourcentage : ', int(100*(nb_varroas-targets[ki])/targets[ki]), ' %', time.ctime())
	detect_names[ki] = cv2.rotate(im_with_keypoints, cv2.cv2.ROTATE_90_CLOCKWISE) # tourne l'image 90° à droite
	cv2.imwrite('final_'+str(ki)+'.jpg',detect_names[ki])     # ecrit le fichier  sur le disque
	print ('  ')
	print ('  ')
	#cv2.imshow('final', detect_names[ki])

cv2.waitKey(0)
cv2.destroyAllWindows()


import re

class StrCalculatorType(object):
    # Data structure for storing the parsed string
    # Abstraction for StrCalculator
    # Let's get rid of a difficult input string,
    # we will keep only the important details.
    def __init__(self, string, delimiters, numstring, numbers):
        self.string = string
        self.delimiters = delimiters
        self.numstring = numstring
        self.numbers = numbers

class StrCalculatorParser(object):
    # string to StrCalculatorType converter
    DEFAULT_DELIMS = ["\n", ","]
    CUSTOM_DELIM_REGEX = re.compile("^([^\d])\n")

    def parse(self, string):
        match = self._check_custom_delim(string)
        if match:
            delimiters, numstring = self._handle_custom_delim(string, match)
        else:
            delimiters, numstring = self._handle_default_delim(string)
        numstring = self.align_delimiters(delimiters, numstring)
        integers = self._split_string(numstring)
        return StrCalculatorType(string, delimiters, numstring, integers)

    def _check_custom_delim(self, string):
        return re.match(self.CUSTOM_DELIM_REGEX, string)

    def _handle_custom_delim(self, string, match):
        delimiters = match.group(1)
        numstring = re.sub(self.CUSTOM_DELIM_REGEX, "", string)
        return delimiters, numstring

    def _handle_default_delim(self, string):
        delimiters = self.DEFAULT_DELIMS
        numstring = string
        return delimiters, numstring

    def align_delimiters(self, delimiters, numstring):
        trantab = numstring.maketrans(''.join(delimiters), "," * len(delimiters))
        numstring = numstring.translate(trantab)
        return numstring

    def _split_string(self, numstring):
        return [int(e) for e in numstring.split(',')]

class StrCalculator(object):
    def add(self, string):
        if string == "":
            return 0
        parser = StrCalculatorParser()
        parsedstr = parser.parse(string)
        self._handle_negative_numbers(parsedstr.numbers)
        return sum(parsedstr.numbers)

    def _handle_negative_numbers(self, numbers):
        negs = [e for e in numbers if e < 0]
        if negs:
            msg = "negatives not allowed: " + ', '.join(str(e) for e in negs)
            raise ValueError(msg)


import math

import numpy as np
from scipy import optimize
from scipy import special
import tensorflow_privacy as tfp

RDP_ORDERS = tuple(map(float, range(2, 21)))
RDP_ORDERS += (22., 24., 28., 32., 64., 128., 256.)
DIV_EPSILON = 1e-22

def log_comb(n, k):
  gammaln = special.gammaln
  return gammaln(n + 1) - gammaln(k + 1) - gammaln(n - k + 1)

def _compute_rdp_subsampled(alpha, gamma, eps, upper_bound=True):
  """Computes RDP with subsampling.

  Reference: http://proceedings.mlr.press/v97/zhu19c/zhu19c.pdf.

  Args:
    alpha: The RDP order.
    gamma: The subsampling probability.
    eps: The RDP function taking alpha as input.
    upper_bound: A bool indicating whether to use Theorem 5 of the referenced
      paper above (if set to True) or Theorem 6 (if set to False).

  Returns:
    The RDP with subsampling.
  """
  if isinstance(alpha, float):
    assert alpha.is_integer()
    alpha = int(alpha)
  assert alpha > 1
  assert 0 < gamma <= 1

  if upper_bound:
    a = [0, eps(2)]
    b = [((1 - gamma)**(alpha - 1)) * (alpha * gamma - gamma + 1),
         special.comb(alpha, 2) * (gamma**2) * (1 - gamma)**(alpha - 2)]

    for l in range(3, alpha + 1):
      a.append((l - 1) * eps(l) + log_comb(alpha, l) +
               (alpha - l) * np.log(1 - gamma) + l * np.log(gamma))
      b.append(3)

  else:
    a = [0]
    b = [((1 - gamma)**(alpha - 1)) * (alpha * gamma - gamma + 1)]

    for l in range(2, alpha + 1):
      a.append((l - 1) * eps(l) + log_comb(alpha, l) +
               (alpha - l) * np.log(1 - gamma) + l * np.log(gamma))
      b.append(1)

  return special.logsumexp(a=a, b=b) / (alpha - 1)

#####################################
######## Gaussian Accounting ########
#####################################

def guass_noise_stddev_direct(epsilon, delta, norm_bound, tol=1.e-12):
  """Compute the stddev for the Gaussian mechanism with the given DP params.

  Calibrate a Gaussian perturbation for differential privacy using the
  analytic Gaussian mechanism of [Balle and Wang, ICML'18].

  Reference: http://proceedings.mlr.press/v80/balle18a/balle18a.pdf.

  Arguments:
    epsilon: Target epsilon (epsilon > 0).
    delta: Target delta (0 < delta < 1).
    norm_bound: Upper bound on L2 global sensitivity (norm_bound >= 0).
    tol: Error tolerance for binary search (tol > 0).

  Returns:
    sigma: Standard deviation of Gaussian noise needed to achieve
      (epsilon,delta)-DP under the given norm_bound.
  """

  exp = math.exp
  sqrt = math.sqrt

  def phi(t):
    return 0.5 * (1.0 + special.erf(float(t) / sqrt(2.0)))

  def case_one(eps, s):
    return phi(sqrt(eps * s)) - exp(eps) * phi(-sqrt(eps * (s + 2.0)))

  def case_two(eps, s):
    return phi(-sqrt(eps * s)) - exp(eps) * phi(-sqrt(eps * (s + 2.0)))

  def doubling_trick(predicate_stop, s_inf, s_sup):
    while not predicate_stop(s_sup):
      s_inf = s_sup
      s_sup = 2.0 * s_inf
    return s_inf, s_sup

  def binary_search(predicate_stop, predicate_left, s_inf, s_sup):
    s_mid = s_inf + (s_sup - s_inf) / 2.0
    while not predicate_stop(s_mid):
      if predicate_left(s_mid):
        s_sup = s_mid
      else:
        s_inf = s_mid
      s_mid = s_inf + (s_sup - s_inf) / 2.0
    return s_mid

  delta_thr = case_one(epsilon, 0.0)

  if delta == delta_thr:
    alpha = 1.0

  else:
    if delta > delta_thr:
      predicate_stop_dt = lambda s: case_one(epsilon, s) >= delta
      function_s_to_delta = lambda s: case_one(epsilon, s)
      predicate_left_bs = lambda s: function_s_to_delta(s) > delta
      function_s_to_alpha = lambda s: sqrt(1.0 + s / 2.0) - sqrt(s / 2.0)

    else:
      predicate_stop_dt = lambda s: case_two(epsilon, s) <= delta
      function_s_to_delta = lambda s: case_two(epsilon, s)
      predicate_left_bs = lambda s: function_s_to_delta(s) < delta
      function_s_to_alpha = lambda s: sqrt(1.0 + s / 2.0) + sqrt(s / 2.0)

    predicate_stop_bs = lambda s: abs(function_s_to_delta(s) - delta) <= tol

    s_inf, s_sup = doubling_trick(predicate_stop_dt, 0.0, 1.0)
    s_final = binary_search(predicate_stop_bs, predicate_left_bs, s_inf, s_sup)
    alpha = function_s_to_alpha(s_final)

  sigma = alpha * norm_bound / sqrt(2.0 * epsilon)
  return sigma

def get_eps_gaussian(q, noise_multiplier, steps, target_delta, orders):
  """Compute eps for the Gaussian mechanism given the DP params."""
  rdp = tfp.compute_rdp(
      q=q, noise_multiplier=noise_multiplier, steps=steps, orders=orders)
  eps, _, _ = tfp.get_privacy_spent(orders, rdp, target_delta=target_delta)
  return eps

def get_gauss_noise_multiplier(target_eps,
                               target_delta,
                               target_sampling_rate,
                               steps,
                               orders=RDP_ORDERS):
  """Compute the Gaussian noise multiplier given the DP params."""

  def get_eps_for_noise_multiplier(z):
    eps = get_eps_gaussian(
        q=target_sampling_rate,
        noise_multiplier=z,
        steps=steps,
        target_delta=target_delta,
        orders=orders)
    return eps

  def opt_fn(z):
    return get_eps_for_noise_multiplier(z) - target_eps

  min_nm, max_nm = 0.001, 1000
  result, r = optimize.brentq(opt_fn, min_nm, max_nm, full_output=True)
  if r.converged:
    return result
  else:
    return -1

############################################################
######## (Distributed) Discrete Gaussian Accounting ########
############################################################

def compute_rdp_discrete_gaussian_simplified(q, l2_scale, tau, dimension, steps,
                                             orders):
  """Compute RDP of the Sampled (Distributed) Discrete Gaussian Mechanism.

  See Proposition 14 / Eq. 17 (Page 16) of the main paper. This function omits
  the RDP term with L1 sensitivity.

  Args:
    q: The sampling rate.
    l2_scale: The l2 scale of the discrete Gaussian mechanism (i.e.,
      l2_sensitivity/stddev). For distributed version, stddev is the noise
      variance after summing all the noise shares.
    tau: The inflation parameter due to adding multiple discrete Gaussians. Set
      to zero when analyzing the the discrete Gaussian mechanism. For the
      distributed discrete Gaussian mechanisn, see Theorem 1.
    dimension: The dimension of the vector query.
    steps: The number of steps.
    orders: An array (or a scalar) of RDP orders, must all be greater than 1. If
      provided orders are not integers, they are rounded down to the nearest
      integer.

  Returns:
    The RDPs at all orders, can be np.inf.
  """
  orders = [int(order) for order in orders]

  def eps(order):
    dim = dimension
    return _compute_rdp_discrete_gaussian_simplified(l2_scale, tau, dim, order)

  if q == 1:
    rdp = np.array([eps(order) for order in orders])
  else:
    rdp = np.array([
        min(_compute_rdp_subsampled(order, q, eps), eps(order))
        for order in orders
    ])

  return rdp * steps

def _compute_rdp_discrete_gaussian_simplified(l2_scale, tau, dimension, order):
  """See Proposition 14 / Eq. 17 (Page 16) of the main paper."""
  assert order >= 1, "alpha must be greater than or equal to 1."
  term_1 = order * (l2_scale**2) / 2.0 + tau * dimension
  term_2 = (order / 2.0) * (l2_scale + math.sqrt(dimension) * tau)**2
  return min(term_1, term_2)

def compute_l2_sensitivy_squared(l2_clip_norm, gamma, beta, dimension):
  """See Theorem 1 of the paper."""
  term_1 = math.pow(l2_clip_norm + gamma * math.sqrt(dimension), 2)
  if beta is None or beta == 0:
    return term_1

  term_2 = math.pow(l2_clip_norm, 2)
  term_2 += 0.25 * math.pow(gamma, 2) * dimension
  term_2 += math.sqrt(2.0 * math.log(1 / beta)) * gamma * (
      l2_clip_norm + 0.5 * gamma * math.sqrt(dimension))
  return min(term_1, term_2)

def get_ddgauss_epsilon(q,
                        noise_stddev,
                        l2_clip_norm,
                        gamma,
                        beta,
                        steps,
                        num_clients,
                        dimension,
                        delta,
                        orders=RDP_ORDERS):
  """Computes the overall epsilon. See Theorem 1 of the paper."""
  variance = math.pow(noise_stddev, 2)
  l2_sensitivity_squared = compute_l2_sensitivy_squared(l2_clip_norm, gamma,
                                                        beta, dimension)
  sq_l2_scale = l2_sensitivity_squared / (num_clients * variance + DIV_EPSILON)
  l2_scale = math.sqrt(sq_l2_scale)

  tau = 0
  for k in range(1, num_clients):
    tau += math.exp(-2 * (variance / (math.pow(gamma, 2) + DIV_EPSILON)) *
                    math.pi**2 * (k * 1.0 / (k + 1)))
  tau *= 10

  rdp = compute_rdp_discrete_gaussian_simplified(q, l2_scale, tau, dimension,
                                                 steps, orders)
  eps, _, _ = tfp.get_privacy_spent(orders, rdp, target_delta=delta)
  return eps

def get_ddgauss_noise_stddev(q,
                             epsilon,
                             l2_clip_norm,
                             gamma,
                             beta,
                             steps,
                             num_clients,
                             dimension,
                             delta,
                             orders=RDP_ORDERS):
  """Selects the local stddev for the distributed discrete Gaussian."""

  def stddev_opt_fn(z):
    cur_epsilon = get_ddgauss_epsilon(q, z, l2_clip_norm, gamma, beta, steps,
                                      num_clients, dimension, delta, orders)
    return (epsilon - cur_epsilon)**2

  stddev_result = optimize.minimize_scalar(stddev_opt_fn)
  if stddev_result.success:
    return stddev_result.x
  else:
    return -1

def get_ddgauss_gamma(q,
                      epsilon,
                      l2_clip_norm,
                      bits,
                      num_clients,
                      dimension,
                      delta,
                      beta,
                      steps,
                      k=1,
                      rho=1,
                      sqrtn_norm_growth=False,
                      orders=RDP_ORDERS):
  """Selects gamma from the given parameters.

  Args:
    q: The sampling factor.
    epsilon: The target overall epsilon.
    l2_clip_norm: The l2 clipping norm for the client vectors.
    bits: The number of bits per coordinate for the aggregated noised vector.
    num_clients: The number of clients per step.
    dimension: The dimension of the vector query.
    delta: The target delta.
    beta: The constant in [0, 1) controlling conditional randomized rounding.
      See Proposition 22 of the paper.
    steps: The total number of steps.
    k: The number of standard deviations of the signal to bound (see Thm. 34 /
      Eq. 61 of the paper).
    rho: The flatness parameter of the random rotation (see Lemma 29).
    sqrtn_norm_growth: A bool indicating whether the norm of the sum of the
      vectors grow at a rate of `sqrt(n)` (i.e. norm(sum_i x_i) <= sqrt(n) * c).
      If `False`, we use the upper bound `norm(sum_i x_i) <= n * c`. See also
      Eq. 61 of the paper.
    orders: The RDP orders.

  Returns:
    The selected gamma.
  """
  if sqrtn_norm_growth:
    n_factor = num_clients
  else:
    n_factor = num_clients**2

  def stddev(x):
    return get_ddgauss_noise_stddev(q, epsilon, l2_clip_norm, x, beta, steps,
                                    num_clients, dimension, delta, orders)

  def mod_min(x):
    return k * math.sqrt(rho / dimension * l2_clip_norm**2 * n_factor +
                         (x**2 / 4.0 + stddev(x)**2) * num_clients)

  def gamma_opt_fn(x):
    return (math.pow(2, bits) - 2 * mod_min(x) / (x + DIV_EPSILON))**2

  gamma_result = optimize.minimize_scalar(gamma_opt_fn)
  if gamma_result.success:
    return gamma_result.x
  else:
    return -1


import pygame as pg
import copy
import time
from m_event import Listener
from m_deathrattle import Deathrattle
from m_effect import Effect
import m_effect
import m_buff

class Card(Listener):

	####methods
	def __init__(o,attack = 0,health = 1,*,name = "", battle_manager = None,taunt = False, divineShield = None):

		Listener.__init__(o)
		o.name = name

		###combat attributes
		o.attack = attack
		o.health = health
		o.max_attack = attack
		o.max_health = health
		o.can_attack = True
		o.divineShield= False
		o.deathrattle_list = []
		o.buff_list = []
		o.ghost = False

		o.taunt = taunt

		###debug attributes
		o.name = name

		###others
		o.owner = None

		###references

		o.battle_manager = None

	### combat methods
	def fight(o,card):
		#print(s.name, " attacked ",card.name)
		o.get_event_manager().spread_event("before_minion_attack", {"source_minion" : o,"target_minion" : card})
		if card.health > 0:
			o.get_event_manager().spread_event("on_minion_attack", {"source_minion" : o, "target_minion" : card})
			o.take_damage(card.attack)
			card.take_damage(o.attack)
			o.get_event_manager().spread_event("after_minion_attack", {"source_minion" : o, "target_minion" : card})
			o.can_attack = False

		if (o.health <= 0):
			o.die()
		if (card.health <= 0):
			card.die()

	def take_damage(o, damage):

		if (o.divineShield and damage > 0):
			o.divineShield = False
			o.battle_manager.event_manager.spread_event("on_divine_shield_lost", {"source_minion" : o})
			return
		o.battle_manager.event_manager.spread_event("on_take_damage", {"target_minion" : o, "damage":damage} )
		o.health -= damage

	def die(o):
		o.get_event_manager().spread_event("on_minion_death", {"source_minion" : o})
		if (o.health <= 0):
			o.ghost = True
		if (o.deathrattle_list != []):
			o.battle_manager.deathrattle_buffer += o.deathrattle_list
		#o.owner.army_before_resolution.remove(o)
		o.get_event_manager().spread_event("after_minion_death", {"source_minion" : o})

	def set_battle_manager(o, battle_manager):
		o.battle_manager = battle_manager
		for deathrattle in o.deathrattle_list:
			deathrattle.set_battle_manager(battle_manager)

	def get_event_manager(o):
		return o.battle_manager.event_manager

	def set_event_manager(o, event_manager):
		o.event_manager = event_manager
		event_manager.add_listener(o)

	def __str__(o):
		return "["+ o.name + "/" + str(o.attack) + "/" + str(o.health) + "/" + str(hex(id(o)))[7:13] + "]"
	###display methods

	def get_position(o,displayer):#where you should draw it
		if o.ghost == True:
			return
		if displayer.display_mode == "arena":
			mZone_length = 3*displayer.win[0]/5
			o.owner.update_minion_pos()
			p_in_army = o.pos
			px = displayer.win[0]/5 + 3*displayer.win[0]/5*p_in_army/o.owner.count_minion_alive()
			py = 0
			if o.owner == displayer.top_player:
				py = displayer.win[1]*1/4
			if o.owner == displayer.bot_player:
				py = displayer.win[1]*2/4
		return [px,py]

	def get_center(o,displayer):
		if o.ghost == True:
			return
		[px,py] = o.get_position(displayer)
		return [px+25,py+50]

	def __repr__(o):
		return o.__str__()

	def get_player(o):
		return o.owner

	def buff(o, attack_bonus, health_bonus, name = "unnamed"):
		o.attack += attack_bonus
		o.health += health_bonus
		o.buff_list.append(m_buff.Buff(attack_bonus, health_bonus, name))

	#prend en param un effet et resout toutes les refs
	def add_deathrattle(o, effect, param = {}):
		deathrattle = Deathrattle(effect, owner = o, battle_manager = o.battle_manager, param = param)
		if (isinstance(effect, Effect)):
			effect.source = deathrattle
		o.deathrattle_list.append(deathrattle)

class Bolvar(Card):
	def __init__(o):
		Card.__init__(o, 1, 7, name='Bolvar')
		o.divineShield = True

		o.listen_to("on_divine_shield_lost", lambda o,p : o.buff(2,0))

class Roi_des_rats(Card):
	def __init__(o):
		Card.__init__(o, 3, 2, name = 'roi des rats')
		effect = m_effect.E_summon(lambda : Rat(), lambda o : o.attack, at = o)

		o.add_deathrattle(effect)
		o.add_deathrattle(lambda o : o.buff(0,2))

		#o.listen_to("on_minion_attack", lambda param : param["source_minion"].buff(0,2))
		#o.listen_to("on_minion_attack", lambda param : param["target_minion"].buff(0,2))
		o.listen_to("on_minion_death", lambda o,param : param["source_minion"].buff(2,0))

	def buff2(o):
		o.buff(2,0)

	def set_summon_number(o, param):
		param["effect"].k = o.attack

class Rat(Card):
	def __init__(o):
		Card.__init__(o,1,1,name = 'rat')

class Ghoul(Card):
	def __init__(o):
		Card.__init__(o, 2, 1, name = 'Ghoul')
		o.add_deathrattle(Deathrattle(effect_type.deal_aoe_damage(1)))

class Scallywag(Card):
	def __init__(o):
		Card.__init__(o, 2, 1, name = 'Scallywag')
		o.add_deathrattle(m_effect.E_summon(lambda : Pirate()))
		o.listen_to("before_minion_attack", lambda o, param: param["target_minion"].take_damage(0))

class Pirate(Card):
	def __init__(o):
		Card.__init__(o, 1, 1, name = 'Pirate')
		o.listen_to("after_summon", lambda o, param : o.owner.command_attack(o) if param["summoned_minion"] == o else None)


from django.db.models import Avg, Count
from django.shortcuts import get_object_or_404
from django_filters.rest_framework import DjangoFilterBackend
from rest_framework import permissions
from rest_framework.pagination import LimitOffsetPagination
from rest_framework.viewsets import ModelViewSet

from reviews.models import Category, Genre, Review, Title
from .filters import TitleFilter
from .mixins import CreateListDestroyViewSet
from .permissions import (AnonimReadOnly,
                          IsSuperUserIsAdminIsModeratorIsAuthor,
                          IsSuperUserOrIsAdminOnly, )
from .serializers import (CategorySerializer, CommentSerializer,
                          GenreSerializer, ReviewSerializer,
                          TitleNotSafeSerializer, TitleSafeSerializer)

class CategoryViewSet(CreateListDestroyViewSet):
    """Вьюсет для создания обьектов класса Category."""

    queryset = Category.objects.all()
    serializer_class = CategorySerializer
    lookup_field = 'slug'

class GenreViewSet(CreateListDestroyViewSet):
    """Вьюсет для создания обьектов класса Genre."""

    queryset = Genre.objects.all()
    serializer_class = GenreSerializer
    lookup_field = 'slug'

class TitleViewSet(ModelViewSet):
    """
    Получить список всех объектов. Права доступа: Доступно без токена
    """
    queryset = Title.objects.annotate(rating=Avg('reviews__score'))
    permission_classes = (AnonimReadOnly | IsSuperUserOrIsAdminOnly,)
    filter_backends = (DjangoFilterBackend,)
    filterset_class = TitleFilter
    pagination_class = LimitOffsetPagination

    def get_serializer_class(self):
        if self.action in ('list', 'retrieve'):
            return TitleSafeSerializer
        return TitleNotSafeSerializer

class ReviewViewSet(ModelViewSet):
    serializer_class = ReviewSerializer
    permission_classes = (
        permissions.IsAuthenticatedOrReadOnly,
        IsSuperUserIsAdminIsModeratorIsAuthor,
    )
    pagination_class = LimitOffsetPagination

    def get_title(self):
        title_id = self.kwargs.get('title_id')
        return get_object_or_404(Title, pk=title_id)

    def get_queryset(self):
        return self.get_title().reviews.all()

    def perform_create(self, serializer):
        serializer.save(
            author=self.request.user,
            title=self.get_title()
        )

class CommentViewSet(ModelViewSet):
    serializer_class = CommentSerializer
    permission_classes = (
        permissions.IsAuthenticatedOrReadOnly,
        IsSuperUserIsAdminIsModeratorIsAuthor,
    )
    pagination_class = LimitOffsetPagination

    def get_review(self):
        review_id = self.kwargs.get('review_id')
        comments = Review.objects.annotate(count=Count("comments"))
        return get_object_or_404(comments, pk=review_id,)

    def get_queryset(self):
        return self.get_review().comments.all()

    def perform_create(self, serializer):
        serializer.save(
            author=self.request.user,
            review=self.get_review())


"""Implémentation du modèle de Nagel-Schreckenberg"""

VITESSE_MAX = 5
MAX_DENSITE = 1

from random import randint

def bool_aleatoire():
    temp = randint(0,1)
    return 0 if temp == 0 else 1

class Voiture:
    #Définition de mon objet
    def __init__(self, nbr_ligne, nbr_colonne):
        self.vitesse = 1

        def destination():
            return randint(0, nbr_ligne-1)

        self.destination_ligne = destination()
        self.destination_colonne = nbr_colonne - 1

    def ajoute_vitesse(self):
        if (self.vitesse < VITESSE_MAX - 1):
            self.vitesse += 1

class Case:
    def __init__(self, y, x, densite = 0):
        self.densite = densite
        self.x = x
        self.y = y
        self.voiture = []

    def ajoute_voiture(self, nbr_ligne, nbr_colonne):
        self.voiture.append(Voiture(nbr_ligne, nbr_colonne))
        self.densite += 1

    def ajoute_densite(self):
        self.densite += 1

    def retire_densite(self):
        if self.densite > 0:
            self.densite -= 1

class Route :
    def __init__(self, nbr_ligne, nbr_colonne):
        self.nbr_ligne = nbr_ligne
        self.nbr_colonne = nbr_colonne
        self.route = [[Case(j,i, randint(0, MAX_DENSITE)) for i in range(nbr_colonne) ] for j in range(nbr_ligne)]
        self.arrive = 0
        self.tour = 0
        self.ajoute_voiture_case()

    def ajoute_voiture_case(self):
        for ligne in self.route:
            for case in ligne:
                for _ in range(case.densite):
                    case.ajoute_voiture(self.nbr_ligne, self.nbr_colonne)

    def avance_voiture(self, case, voiture1, case_suivante):
        case_suivante.voiture.append(voiture1)
        case_suivante.ajoute_densite()
        case.voiture.remove(voiture1)
        case.retire_densite()

    def avance_direction(self, voiture, case):
        if voiture.destination_ligne == case.y:
            if case.x + 1 == self.nbr_colonne:
                self.arrive += 1
                case.voiture.remove(voiture)
                return ;
            self.avance_voiture( case, voiture, self.route[case.y][case.x+1])
        elif voiture.destination_ligne < case.y:
            self.avance_voiture(case, voiture, self.route[case.y - 1][case.x])
        else :
            self.avance_voiture(case, voiture, self.route[case.y + 1][case.x])

    def jouer_tour(self):
        self.tour += 1
        for i in range(self.nbr_ligne):
            for j in range(self.nbr_colonne):
                case = self.route[self.nbr_ligne - i - 1][self.nbr_colonne - j - 1]
                for voiture in case.voiture:
                    self.avance_direction(voiture, case)

    def check_tous_arrive(self):
        for ligne in self.route:
            for case in ligne:
                if case.voiture != []:
                    return False
        return True

def main(nbr_ligne, nbr_colonne):
    route1 = Route(nbr_ligne, nbr_colonne)
    while not route1.check_tous_arrive():
        route1.jouer_tour()
        #while(not route1.check_tous_arrive()):
        #route1.jouer_tour()

    print(route1.arrive,"voitures arrivées en", route1.tour, "tours")

main(100, 100)


import sqlite3

con =sqlite3.connect("sample.db")
cursor = con.cursor()

# User名を指定して、Aeg （年齢）を更新するprogram

# ユーザーに何をしてもらいたいか？
target_name = input("whose 'age' do you want to update?")
new_age = input("Tell me new age:")
print(target_name, new_age)

# SQL 文を作成
# 文字列を使って SQL を作成するという事は脆弱性があるので注意する
# SQL ingection という攻撃方法 : new age の後にセミコロン「; DROP TABLE User;」 で User table data を消されてしまう
# update_query = f"UPDATE User SET age = {new_age} WHERE name = '{target_name}'"
# cursor.executescript(update_query)

# User の input で update する。ほとんどそのような場合だが…
# 上記のような攻撃を防ぐ為に必ず、 execute method を使って プレイスフォルダーを使って値を tuple で渡す
update_query = 'UPDATE User SET age = ? WHERE name = ?'
cursor.execute(update_query, (new_age, target_name))
con.commit()


import os
from setuptools import setup

DESC = "Some decorators and classes to make working with django projects easier."

try:
    with open('readme.rst') as file:
        long_description = file.read()
except IOError:
    long_description = DESC

setup(
    name = "django-drapes",
    version = "0.1.2",
    author = "Ulas Tuerkmen",
    author_email = "afroisalreadyinu@gmail.com",
    description = DESC,
    long_description=long_description,
    install_requires = ['formencode',
                        'decorator',
                        'django',
                        'coverage',
                        'nose',
                        'mock'],
    py_modules=['django_drapes'],
    classifiers=['Framework :: Django',],
)


import time
import csv

LOCKED = False

def wait_for_lock(func):
    ''' Decorator function to ensure that functions
        don't read/write to CSV file at the same time'''
    def wrapper(*args, **kwargs):
        global LOCKED
        for i in range(100):
            if not LOCKED:
                LOCKED = True
                val = func(*args, **kwargs)
                LOCKED = False
                return val
            else:
                time.sleep(2)
    return wrapper

@wait_for_lock
def create_csv() -> None:
    ''' Creates a CSV file called ping_data.csv
        with 3 columns -> 'blockNumber', 'transactionHash', 'PongStatus'
        Contains no entries -> use add_to_ping_csv() to add data to file '''

    with open('ping_data2.csv', 'w') as new_file:
        csv_writer = csv.writer(new_file, delimiter=',')
        csv_writer.writerow(['blockNumber', 'transactionHash', 'PongStatus'])

@wait_for_lock
def add_to_ping_csv(ping_data:tuple) -> None:
    ''' Takes in a tuple ('blockNumber', 'transactionHash')
        Checks to see if the transactionHash exists in the CSV
        If it does not exist -> add ping_data + No Pong to CSV'''
    _, transactionHash = ping_data
    with open('ping_data2.csv', 'r') as csv_file:
        csv_reader = csv.DictReader(csv_file)
        if transactionHash not in [line['transactionHash'] for line in csv_reader]:
            add_row = list(ping_data)
            add_row.append("No Pong")
            with open('ping_data2.csv', 'a', newline='') as csv_file_append:
                csv_writer = csv.writer(csv_file_append)
                csv_writer.writerow(add_row)
                print(f"Adding {ping_data} to csv file")

@wait_for_lock
def get_last_entry() -> list:
    ''' Returns the last row of the ping_data.csv as a tuple
        (blockNumber, transactionHash, PongStatus) if there are
        entries otherwise returns None'''

    with open('ping_data2.csv', 'r', newline='') as csv_file:
        csv_reader = csv.DictReader(csv_file)
        lines = [list(line.values()) for line in csv_reader]
        return lines[-1] if lines else None

@wait_for_lock
def get_first_unPonged() -> tuple:
    ''' Returns the index and transactionHash of
        first instance where PongStatus is 'No Pong'
        ============================================
        Checks to see if there are any entries in the CSV
        then if there are any 'No Pong' instances if not
        waits 2 minutes and re-checks '''
    for _ in range(20):
        with open('ping_data2.csv', 'r', newline='') as csv_file:
            csv_reader = csv.DictReader(csv_file)
            lines = [list(line.values()) for line in csv_reader if line['PongStatus'] == 'No Pong']
            return lines[0] if lines else time.sleep(120)

@wait_for_lock
def update_pongStatus(index:int, tx_hash:str) -> None:
    ''' Updates the PongStatus & Nonce in the CSV file after transaction'''
    pass

if __name__ == '__main__':

    # create_csv()

    # pd = (987654321, '0x8b7307d22786478e4ebcd8ba5c6ef8042fa20c818d42ad7eeb3e0c18e604795i')
    # add_to_ping_csv(pd)

    unponged = get_first_unPonged()
    print(unponged)