import numpy as np
import sys
import states

class AStar:
    cost = None
    heuristic = None
    _cache = None
    shouldCache = None

    def __init__(self, heuristic, cost=None, shouldCache=False):
        self.heuristic = heuristic
        self.shouldCache = shouldCache
        self.cost = cost

        # Handles the cache. No reason to change this code.
        if self.shouldCache:
            self._cache = {}

    # Get's from the cache. No reason to change this code.
    def _getFromCache(self, problem):
        if self.shouldCache:
            return self._cache.get(problem)

        return None

    # Get's from the cache. No reason to change this code.
    def _storeInCache(self, problem, value):
        if not self.shouldCache:
            return

        self._cache[problem] = value

    # Run A*
    def run(self, problem):
        # Check if we already have this problem in the cache.
        # No reason to change this code.
        source = problem.initialState
        if self.shouldCache:
            res = self._getFromCache(problem)

            if res is not None:
                return res

        # Initializes the required sets
        closed_set = set()  # The set of nodes already evaluated.
        parents = {}  # The map of navigated nodes.

        # Save the g_score and f_score for the open nodes
        g_score = {source: 0}
        open_set = {source: self.heuristic.estimate(problem, problem.initialState)}

        developed = 0

        # TODO : Implement astar.
        # Tips:
        # - To get the successor states of a state with their costs, use: problem.expandWithCosts(state, self.cost)
        # - You should break your code into methods (two such stubs are written below)
        # - Don't forget to cache your result between returning it - TODO

        while open_set:
            current = self._getOpenStateWithLowest_f_score(open_set)
            open_set.pop(current)
            closed_set.add(current)
            #if current.junctionIdx == problem.target.junctionIdx:
            if problem.isGoal(current):
                path = self._reconstructPath(parents, current, problem.initialState)
                #path.insert(0,problem.initialState)
                res = (path, g_score[current],
                        self.heuristic.estimate(problem, problem.initialState),
                        developed)
                self._storeInCache(problem,res)
                return res
            developed += 1
            for s in problem.expandWithCosts(current, self.cost):
                new_g = g_score[current] + s[1]
                # old_node = open_set[s[0]]
                if s[0] in open_set:
                    old_node = s[0]
                    if new_g < g_score[old_node]:
                        g_score[old_node] = new_g
                        parents[old_node] = current
                        open_set[old_node] = new_g + self.heuristic.estimate(problem, s[0])
                        # else
                elif s[0] in closed_set:
                    # old_node = closed_set[s]
                    old_node = s[0]
                    if new_g < g_score[old_node]:
                        g_score[old_node] = new_g
                        parents[old_node] = current
                        open_set[old_node] = new_g + self.heuristic.estimate(problem, s[0])
                        closed_set.remove(old_node)
                else:
                    open_set[s[0]] = s[1] + g_score[current] + self.heuristic.estimate(problem, s[0])
                    g_score[s[0]] = s[1] + g_score[current]
                    parents[s[0]] = current

                    # TODO update g_score, develpoed, parents

        # TODO : VERY IMPORTANT: must return a tuple of (path, g_score(goal), h(I), developed)
        return ([], -1, -1, developed)

    def _getOpenStateWithLowest_f_score(self, open_set):
        # TODO : Implement
        minState = None
        min = 0
        for s in open_set:
            if minState is None or open_set[s] < min :
                min = open_set[s]
                minState = s
        return minState
        # raise NotImplementedError

    # Reconstruct the path from a given goal by its parent and so on
    def _reconstructPath(self, parents, goal, initial):
        # TODO : Implement
        currentNode = goal
        path = [goal]
        while currentNode is not initial:
            path.insert(0, parents[currentNode])
            currentNode=parents[currentNode]
        return path
        # raise NotImplementedError


#Given the results of oligo enrichments, as calculated by DESeq2, define windows
#of enriched oligos that encompass active RNA elements.  Then, once you have
#windows, define minimal sequences within those windows that represent the
#minimal active element. These would be sequences that are in common to every
#oligo within the window.

#Windows are defined by looking for an enriched oligo and extending a window if either
#(A) the next oligo is sig or
#(B) there are x sig oligos in a row after a gap of x oligos

#After defining these primary windows, two windows are merged together if
#the distance between them is less than  (width of the narrow window + 2)
#After merging, any window less than 10 oligos long is removed

import os
import pandas as pd
import sys
import gffutils
from itertools import groupby
from itertools import product
from operator import itemgetter
import gzip
from Bio import SeqIO
import pysam

def simpleroligos2genome(gff):
    #Just make a table that relates oligo position in the UTR (i.e. 1, 2, 3, 4, etc.)
    #to the start genomecoord of the oligo

    oligocoords = {} #{utr : {oligoposition : genomecoordofstart}}
    oligostrands = {} #{utr : strand}
    oligochrms = {} #{utr: chrom}

    print('Indexing gff...')
    gff_fn = gff
    db_fn = os.path.abspath(gff_fn) + '.db'
    if os.path.isfile(db_fn) == False:
        gffutils.create_db(gff_fn, db_fn, merge_strategy = 'merge', verbose = True)

    db = gffutils.FeatureDB(db_fn)
    print('Done indexing!')

    utrs = db.features_of_type('UTR')

    for utr in utrs:
        utrid = str(utr.attributes['gene_name'][0])
        if utrid not in oligocoords:
            oligocoords[utrid] = {}
        if utrid not in oligostrands:
            oligostrands[utrid] = utr.strand
        if utrid not in oligochrms:
            oligochrms[utrid] = utr.chrom
        for oligo in db.children(utr, featuretype = 'oligo'):
            oligoid = int(str(oligo.id).split('.')[1])
            if oligo.strand == '+':
                genomecoord = int(oligo.start)
            elif oligo.strand == '-':
                genomecoord = int(oligo.stop)
            oligocoords[utrid][oligoid] = genomecoord
            if utrid == 'Afap1l1' and oligoid == 1:
                print('yo')

    #turn into df
    df = pd.concat({k: pd.DataFrame.from_dict(v, 'index') for k, v in oligocoords.items()}, axis = 0)
    #get indexes out as column names
    df.reset_index(level = 1, inplace = True)
    df.reset_index(level = 0, inplace = True)
    df.columns = ['genename', 'oligopos', 'genomecoord']
    df['strand'] = [oligostrands[x] for x in df['genename']]
    df['chrm'] = [oligochrms[x] for x in df['genename']]
    print(df.head())

    return df

def oligos2genome(gff):
	#We need some way to relate oligo positions, namely through their ID's (e.g. ENSMUSG00000025352.256|Gdf11)
	#to positions in the genome that they cover. One way to do this would be to, for each oligo,
	#make a list of coordinates that are tiled by the oligo: {utr : [list of exonic coordinates covered by utr's oligos]}
	#then, from that, make dictionary of {utr : {utr nt (as defined by getntcoverage (this goes from 1 to len(utr))) : genome coord}}

	#Gffs are annotations of oligo positions and were made by OligoPools_shortstep_260nt.py

	utrcoords = {} #{utr : [list of exonic coordinates covered by utr's oligos]}

	print('Indexing gff...')
	gff_fn = gff
	db_fn = os.path.abspath(gff_fn) + '.db'
	if os.path.isfile(db_fn) == False:
		gffutils.create_db(gff_fn, db_fn, merge_strategy = 'merge', verbose = True)

	db = gffutils.FeatureDB(db_fn)
	print('Done indexing!')

	utrs = db.features_of_type('UTR')

	for utr in utrs:
		utrid = str(utr.attributes['gene_name'][0])
		utrexoniccoords = []
		for oligo in db.children(utr, featuretype = 'oligo', level = 1):
			#Does this oligo cross a splice junction (i.e. does it have junctionpiece children?)
			if len(list(db.children(oligo, featuretype = 'junctionpiece'))) > 0:
				#yes there is a junctionpiece
				for jp in db.children(oligo, featuretype = 'junctionpiece'):
					jpnt = list(range(jp.start, jp.end + 1))
					utrexoniccoords += jpnt
			elif len(list(db.children(oligo, featuretype = 'junctionpiece'))) == 0:
				#no there is not a junctionpiece
				oligont = list(range(oligo.start, oligo.end + 1))
				utrexoniccoords += oligont

		#there's gonna be a lot of redundancy in utrexoniccoords because the oligos overlap
		utrexoniccoords = list(set(utrexoniccoords))
		if utr.strand == '+':
			utrexoniccoords = list(sorted(utrexoniccoords))
		elif utr.strand == '-':
			utrexoniccoords = list(reversed(sorted(utrexoniccoords)))

		utrcoords[utrid] = utrexoniccoords

	#OK now we need to go from utrcoords to utr_oligocoords {utr : {utr nt (as defined by getntcoverage (this goes from 1 to len(utr))) : genome coord}}

	utr_oligocoords = {} #{utr : {utr nt (as defined by getntcoverage (this goes from 1 to len(utr))) : genome coord}}
	for utr in utrcoords:
		genomecoords = utrcoords[utr]
		oligocoords = list(range(len(genomecoords) + 1))
		oligocoords2genomecoords = dict(zip(oligocoords, genomecoords))
		utr_oligocoords[utr] = oligocoords2genomecoords

	#turn utr_oligocoords into a df and return the df
	df = pd.concat({k: pd.DataFrame.from_dict(v, 'index') for k, v in utr_oligocoords.items()}, axis = 0)
	#get indexes out as column names
	df.reset_index(level = 1, inplace = True)
	df.reset_index(level = 0, inplace = True)
	df.columns = ['genename', 'utrcoord', 'genomecoord']

	return df

def definewindows(deseqtable, windowlengthminimum):
    enrichedwindows = {} #{gene : [(windowstart, windowstop)]}
    #The input for this function is a modified DESeq2 output table
    #It contains the following columns:
    #ensid, genename, baseMean, log2FoldChange, lfcSE, stat, pvalue, padj, sig, neuritelog2FC, expected
    #sig is either 'soma', 'neurite', or 'no'

    #Read file
    df = pd.read_csv(deseqtable, sep = '\t', header = 0, index_col = None)
    #ensid is of the form ENSMUSG00000.0
    #the number after the dot is the position of the oligo along the UTR
    df[['ensid', 'oligopos']] = df['ensid'].str.split('.', n = 1, expand = True)
    #Reorder columns
    df = df[['ensid', 'oligopos', 'genename', 'baseMean', 'log2FoldChange', 'lfcSE', 'stat', 'pvalue', 'padj', 'sig', 'neuritelog2FC', 'expected']]
    #Get rid of columns we don't need
    df = df[['genename', 'oligopos', 'sig']]
    #Convert oligopos to integer
    df = df.astype({'oligopos' : int})
    #Add in the start nucleotide position (0-based) of each oligo relative to the beginning of the first oligo for this gene
    #Oligos are 260 nt long, 4 nt step size
    oligopos = df.oligopos.tolist()
    utrcoord = [(x - 1) * 4 for x in oligopos]
    df = df.assign(utrcoord = utrcoord)
    #Arrange by genename then oligopos
    df = df.sort_values(by = ['genename', 'oligopos'], axis = 0)

    #OK, now for each gene, define windows of significance
    #Walk along UTR. oligopos = x. If oligo at x sig == 'neurite' oligo, open window
    #If x + 1 oligo is sig, extend window and repeat.
    #If not, go until you find a sig. Distance between current pos and next sig is y
    #If -all- of the oligos between x + y and x + 2y are sig, then extend window and go back to the beginning.
    #If not, window ends.
    #If window length passes threshold, keep it.

    genes = df.genename.tolist()
    genes = list(set(genes))
    genedfs = [] #list of individual genedfs
    for gene in genes:
        genedf = df.loc[df['genename'] == gene]
        alloligopos = genedf.oligopos.tolist()
        sigoligopos = genedf.loc[genedf['sig'] == 'neurite'].oligopos.tolist()

        inwindow = False
        oligoposinwindow = [] #oligo positions in -any- window
        currentidx = 0
        for idx, oligopos in enumerate(alloligopos):
            #print(gene, idx, oligopos, inwindow, oligoposinwindow)
            if idx < currentidx:
                continue
            if inwindow == False:
                if oligopos not in sigoligopos:
                    continue
                elif oligopos in sigoligopos:
                    inwindow = True
                    oligoposinwindow.append(oligopos)
            elif inwindow == True:
                if oligopos in sigoligopos: #if we are in a window and the next oligo is also sig, just extend the window
                    oligoposinwindow.append(oligopos)
                    continue
                elif oligopos not in sigoligopos:
                    findasigoligoaftergap = False #are we able to find a sig oligo within 10 oligos of gap start
                    for i in range(1, 11):
                        doesthisgappass = False #does this gap fit the requirements of x sig oligos after a gap of x
                        try:
                            oligoinquestion = alloligopos[idx + i]
                            if oligoinquestion in sigoligopos:# if we found a gap that is < 10
                                findasigoligoaftergap = True
                                gapendidx = alloligopos.index(oligoinquestion)
                                #All oligos from gapidx up to (but not including) gapidx + i must be sig in order for window to continue
                                postgapsigoligos = [] #booleans for the sig-ness of the required post-gap oligos
                                doesthisgappass = False #is this gap valid? Does it have the required number of consecutive sig oligos after the gap?
                                for j in range(0, i):
                                    try:
                                        x = alloligopos[gapendidx + j]
                                        if x in sigoligopos:
                                            postgapsigoligos.append(True)
                                        else:
                                            postgapsigoligos.append(False)
                                    except IndexError:
                                        postgapsigoligos.append(False)
                                if all(postgapsigoligos): #are all items in postgapsigoligos True
                                    doesthisgappass = True
                                    break
                                else:
                                    doesthisgappass = False

                        except IndexError:
                            inwindow = False
                            doesthisgappass = False
                            break

                        if doesthisgappass == True:
                            currentidx = idx + i + i
                            #Append all oligos from idx up to (but not including) currentidx to oligoposinwindow
                            for k in range(idx, currentidx):
                                oligoposinwindow.append(alloligopos[k])
                            inwindow = True
                            break

                    #If we didn't find a sig oligo within 10 positions of the gapstart
                    if findasigoligoaftergap == False:
                        inwindow = False

        #Turn this list of positions in a window into a list of tuples of windowstart/windowstop
        #https://stackoverflow.com/questions/2154249/identify-groups-of-continuous-numbers-in-a-list
        windows = [] #tuples of (windowstart, windowstop)
        for k, g in groupby(enumerate(oligoposinwindow), lambda x:x[0] - x[1]):
            group = list(map(itemgetter(1), g))
            windows.append((group[0], group[-1]))

        #Merge windowA and windowB if they are less than min(windowAlength, windowBlength) + 2 apart
        mergedwindows = [] #list of positions in merged windows
        for windowA in windows:
            windowAstart = windowA[0]
            windowAend = windowA[1]
            windowAlength = (windowAend - windowAstart) + 1
            wasmerged = False
            for windowB in windows:
                windowBstart = windowB[0]
                windowBend = windowB[1]
                windowBlength = (windowBend - windowBstart) + 1
                allowabledistance = min([windowAlength, windowBlength]) + 2
                if abs(windowBstart - windowAend) <= allowabledistance or abs(windowAstart - windowBend) <= allowabledistance:
                    mergedwindow = (min(windowAstart, windowBstart), max(windowAend, windowBend))
                    for i in range(mergedwindow[0], mergedwindow[1] + 1):
                        mergedwindows.append(i)
                    wasmerged = True

            #if this window was not merged with anything, just put the original window into mergedwindows
            if not wasmerged:
                for i in range(windowAstart, windowAend + 1):
                    mergedwindows.append(i)

        #remove duplicates and sort mergedwindows
        mergedwindows = sorted(list(set(mergedwindows)))

        #Break consective windows into chunks as above
        windows = [] #tuples of (windowstart, windowstop)
        for k, g in groupby(enumerate(mergedwindows), lambda x:x[0] - x[1]):
            group = list(map(itemgetter(1), g))
            windows.append((group[0], group[-1]))

        #Filter for windows that are at least windowlengthminimum long
        windows = [window for window in windows if ((window[1] - window[0]) + 1) >= windowlengthminimum]

        #Add data about windows to dictionary
        positionsinawindow = []
        enrichedwindows[gene] = []
        for window in windows:
            enrichedwindows[gene].append(window)
            for position in range(window[0], window[1] + 1):
                positionsinawindow.append(position)

        #Now add a new column to the df that tells whether or not that position is in a window
        inwindow = []
        for i in alloligopos:
            if i in positionsinawindow:
                inwindow.append('yes')
            else:
                inwindow.append('no')

        #Add column to df
        genedf = genedf.assign(inwindow = inwindow)
        genedfs.append(genedf)

    #Concatentate all genedfs back together
    allgenedfs = pd.concat(genedfs, axis = 0)

    return enrichedwindows, allgenedfs

def getminimalseqs(enrichedwindows, oligogff, cushion):
    #Given a set of oligo windows provided by definewindows(), get the minimal sequence
    #within those windows associated with localization. If windowmode == 'intersect', these will be defined as the sequence
    #in common to all oligos within the window. If windowmode == 'union', these will be defined as the sequence
    #in any oligo within the window

    #enrichedwindows has the following structure:
    #{genename : [(first oligo in window1, last oligo in window1), (first oligo in window2, last oligo in window2)]}
    #In this dictionary, oligos are defined by their position within the UTR, starting with 1
    #This designations correspond to the numbers after the ENS id and '.' in the oligo gff

    minimalseqs = [] #[[chrm, 'mm10', 'minimalseq', start, stop, '.', strand, attributes]]
    windowoligosgff = []

    print('Indexing gff...')
    gff_fn = oligogff
    db_fn = os.path.abspath(gff_fn) + '.db'
    if os.path.isfile(db_fn) == False:
        gffutils.create_db(gff_fn, db_fn, merge_strategy = 'merge', verbose = True)

    db = gffutils.FeatureDB(db_fn)
    print('Done indexing!')

    for gene in enrichedwindows:
        windowcounter = 0
        for window in enrichedwindows[gene]:
            windowcounter +=1
            coverednt = [] #[[all nt covered by oligo1], [all nt covered by oligo2], ...]
            firstoligo = window[0]
            lastoligo = window[1]
            utrs = db.features_of_type('UTR')
            for utr in utrs:
                genename = utr.attributes['gene_name'][0]
                if genename == gene:
                    chrm = str(utr.chrom)
                    strand = str(utr.strand)
                    ensid = str(utr.id).split('.')[0]
                    for oligo in db.children(utr, featuretype = 'oligo', level = 1):
                        oligonumber = int(str(oligo.id).split('.')[1])
                        #if this is one of the oligos we are after
                        if oligonumber >= firstoligo and oligonumber <= lastoligo:
                            #is this a oneexon oligo?
                            if oligo.attributes['oligo_type'][0] == 'regular_oneexon':
                                coverednt.append(list(range(oligo.start, oligo.end + 1)))
                                windowoligosgff.append([str(oligo.chrom), 'sigWindowOligos', 'oligo', str(oligo.start), str(oligo.stop), '.', oligo.strand, '.', 'ID={0};gene_name={1};oligo_type={2}'.format(str(oligo.id), str(oligo.attributes['gene_name'][0]), 'regular_oneexon')])
                            elif oligo.attributes['oligo_type'][0] == 'junction':
                                thisoligosnt = []
                                windowoligosgff.append([str(oligo.chrom), 'sigWindowOligos', 'oligo', str(oligo.start), str(oligo.stop), '.', oligo.strand, '.', 'ID={0};gene_name={1};oligo_type={2}'.format(str(oligo.id), str(oligo.attributes['gene_name'][0]), 'junction')])
                                for junctionpiece in db.children(oligo, featuretype = 'junctionpiece'):
                                    windowoligosgff.append([str(oligo.chrom), 'sigWindowOligos', 'junctionpiece', str(junctionpiece.start), str(junctionpiece.stop), '.', oligo.strand, '.', 'ID={0};gene_name={1};oligo_type={2};Parent={3}'.format(str(junctionpiece.id), str(junctionpiece.attributes['gene_name'][0]), 'junction', str(junctionpiece.attributes['Parent'][0]))])
                                    junctionpiecent = list(range(junctionpiece.start, junctionpiece.end + 1))
                                    thisoligosnt += junctionpiecent
                                coverednt.append(thisoligosnt)

            #now get the nt in the window
            if gene == 'Cplx2':
                windowmode = 'union'
            else:
                windowmode = 'union'
            if windowmode == 'intersect':
                commonnt = set(coverednt[0])
                for x in coverednt[1:]:
                    commonnt.intersection_update(x)
                commonnt = sorted(list(commonnt))
            elif windowmode == 'union':
                setlist = [set(x) for x in coverednt]
                unionnt = set().union(*setlist)
                commonnt = sorted(list(unionnt))

            #Does this minimal element span an intron?
            #Turn this list of positions in a window into a list of tuples of windowstart/windowstop
            #https://stackoverflow.com/questions/2154249/identify-groups-of-continuous-numbers-in-a-list
            #length of minimalelementwindows is the number of exons contained in this minimal element
            minimalelementwindows = [] #tuples of (windowstart, windowstop)
            for k, g in groupby(enumerate(commonnt), lambda x:x[0] - x[1]):
                group = list(map(itemgetter(1), g))
                minimalelementwindows.append((group[0], group[-1]))

            #if this whole minimal element is contained within one exon
            if len(minimalelementwindows) == 1:
                minimalseq = [chrm, 'mm10', 'minimalseq', str(minimalelementwindows[0][0] - cushion), str(minimalelementwindows[-1][1] + cushion), '.', strand, '.', 'ID={0}.minimalseq{1};gene_name={2};element_type={3}'.format(ensid, windowcounter, gene, 'singleexon')]
                minimalseqs.append(minimalseq)
            #if it spans multiple exons
            elif len(minimalelementwindows) > 1:
                minimalelementexoncounter = 0
                wholewindow = [chrm, 'mm10', 'minimalseq', str(minimalelementwindows[0][0] - cushion), str(minimalelementwindows[-1][1] + cushion), '.', strand, '.', 'ID={0}.minimalseq{1};gene_name={2};element_type={3}'.format(ensid, windowcounter, gene, 'multiexon')]
                minimalseqs.append(wholewindow)
                for x in minimalelementwindows:
                    minimalelementexoncounter +=1
                    minimalseq = [chrm, 'mm10', 'minimalseqexon', str(x[0]), str(x[1]), '.', strand, '.', 'ID={0}.minimalseq{1}.exon{2};gene_name={3};element_type={4};Parent={5}'.format(ensid, windowcounter, minimalelementexoncounter, gene, 'multiexon', ensid + '.minimalseq' + windowcounter)]
                    minimalseqs.append(minimalseq)

    return minimalseqs, windowoligosgff

def makenonwindowgff(oligogff, windowgff):
    #Given a gff of oligo positions, first get rid of any sequence that contains CDS. This happens because we included sequence upstream
    #of the start codon to get good coverage of the beginning of the UTR. As of now, this removal is done by hand.
    #Then, make a gff that contains UTR regions that are covered by oligos but NOT by a significant window. This is in preparation for looking a sequence content, etc.
    #of UTR regions in significant windows vs. not in significant windows.
    #Sig windows are minimal seqs as defined by getminimalseqs

    #TODO: get these coordinates from mm10 gff
    firstfullyUTRoligo = {} #{genename : coordinate of first UTR nt in UTRs covered by oligos}
    firstfullyUTRoligo['Trak2'] = 58903503
    firstfullyUTRoligo['Fnbp4'] = 90779652
    firstfullyUTRoligo['Trp53inp2'] = 155386793
    firstfullyUTRoligo['Akap12'] = 4357932
    firstfullyUTRoligo['Gdf11'] = 128885105
    firstfullyUTRoligo['Net1'] = 3884069
    firstfullyUTRoligo['Cplx2'] = 54379710
    firstfullyUTRoligo['Paxbp1'] = 91015060
    firstfullyUTRoligo['Kif5b'] = 6208204
    firstfullyUTRoligo['Afap1l1'] = 61731286
    firstfullyUTRoligo['Malat1'] = 5802671
    firstfullyUTRoligo['Cdc42bpg'] = 6324574
    firstfullyUTRoligo['Ogt'] = 101682331
    firstfullyUTRoligo['Rab13'] = 90225584

    print('Indexing gff...')
    gff_fn = oligogff
    oligodb_fn = os.path.abspath(gff_fn) + '.db'
    if os.path.isfile(oligodb_fn) == False:
        gffutils.create_db(gff_fn, oligodb_fn, merge_strategy = 'merge', verbose = True)

    oligodb = gffutils.FeatureDB(oligodb_fn)
    print('Done indexing!')

    print('Indexing gff...')
    gff_fn = windowgff
    windowdb_fn = os.path.abspath(gff_fn) + '.db'
    if os.path.isfile(windowdb_fn) == False:
        gffutils.create_db(gff_fn, windowdb_fn, merge_strategy = 'merge', verbose = True)

    windowdb = gffutils.FeatureDB(windowdb_fn)
    print('Done indexing!')

    #Get nucelotides that are in a minimal seq window
    minimalseqnt = {} #{gene : [nt in minimalseq]}
    minimalseqs = windowdb.features_of_type('minimalseq')
    for minimalseq in minimalseqs:
        gene = minimalseq.attributes['gene_name'][0]
        if gene not in minimalseqnt:
            minimalseqnt[gene] = []
        nt = list(range(minimalseq.start, minimalseq.end + 1))
        minimalseqnt[gene] += nt

    oligononwindownt = {}  #{gene : [nt that are not in sig window]}
    genename2ensid = {} #{gene_name : ensid}

    oligos = oligodb.features_of_type('oligo')
    for oligo in oligos:
        gene = oligo.attributes['gene_name'][0]
        if gene not in oligononwindownt:
            oligononwindownt[gene] = []
        strand = oligo.strand
        utrboundary = firstfullyUTRoligo[gene]
        nt = list(range(oligo.start, oligo.end + 1))
        #Get rid of any nt outside of nt boundary
        if strand == '+':
            nt = [x for x in nt if x >= utrboundary]
        elif strand == '-':
            nt = [x for x in nt if x <= utrboundary]
        #Get rid of any nt in a sig window
        try:
            windownt = minimalseqnt[gene]
        except KeyError: #this gene wasn't in minimalseqgff (probably because it didn't have a minimal seq)
            windownt = []
        nt = [x for x in nt if x not in windownt]
        oligononwindownt[gene] += nt

        ensid = str(oligo.id).split('.')[0]
        genename2ensid[gene] = ensid

    #Go through oligononwindownt and break it up into contiguous chunks
    oligononwindownt_chunks = {} #{gene : [[chunk1start, chunk1stop], [chunk2start, chunk2stop], ...]}
    #Break consective windows into chunks as above
    for gene in oligononwindownt:
        nt = sorted(list(set(oligononwindownt[gene])))
        windows = [] #tuples of (windowstart, windowstop)
        for k, g in groupby(enumerate(nt), lambda x:x[0] - x[1]):
            group = list(map(itemgetter(1), g))
            windows.append((group[0], group[-1]))
        oligononwindownt_chunks[gene] = windows

    #Write nonwindownt to gff
    with open('nonminimalseqs.gff', 'w') as outfh:
        for gene in oligononwindownt_chunks:
            ensid = genename2ensid[gene]
            utrensid = genename2ensid[gene] + '.UTR'
            chrm = oligodb[utrensid].chrom
            strand = oligodb[utrensid].strand
            spanstart = str(oligononwindownt_chunks[gene][0][0])
            spanend = str(oligononwindownt_chunks[gene][-1][1])
            outfh.write(('\t').join([chrm, 'mm10', 'nonminimalseqspan', spanstart, spanend, '.', strand, '.', 'ID={0}.nonminimalseqspan;gene_name={1}'.format(ensid, gene)])+ '\n')
            chunkcounter = 0
            for chunk in oligononwindownt_chunks[gene]:
                chunkcounter +=1
                chunkstart = str(chunk[0])
                chunkend = str(chunk[1])

                outfh.write(('\t').join([chrm, 'mm10', 'nonminimalseqchunk', chunkstart, chunkend, '.', strand, '.', 'ID={0}.nonminimalseqchunk{1};gene_name={2};Parent={3}.nonminimalseqspan'.format(ensid, chunkcounter, gene, ensid)]) + '\n')

    os.remove(oligodb_fn)
    os.remove(windowdb_fn)

def gff2seq(gff, feature, genomefasta, outfilename):
    #Given a gff and the featurename to look for, output a fasta

    print('Indexing gff...')
    gff_fn = gff
    db_fn = os.path.abspath(gff_fn) + '.db'
    if os.path.isfile(db_fn) == False:
        gffutils.create_db(gff_fn, db_fn, merge_strategy = 'merge', verbose = True)

    db = gffutils.FeatureDB(db_fn)
    print('Done indexing!')

    with open(outfilename, 'w') as outfh:
        with gzip.open(genomefasta, 'rt') as genomefh:
            print('Indexing genome...')
            seq_dict = SeqIO.to_dict(SeqIO.parse(genomefh, 'fasta'))
            print('Done!')
            for f in db.features_of_type(feature):
                chrm = str(f.chrom)
                start = f.start
                end = f.end
                strand = f.strand
                seqid = str(f.id) + '_' + str(f.attributes['gene_name'][0])
                outfh.write('>' + seqid + '\n')
                if strand == '+':
                    seq = seq_dict[chrm].seq[start - 1 : end].upper()
                elif strand == '-':
                    seq = seq_dict[chrm].seq[start - 1 : end].reverse_complement().upper()
                outfh.write(str(seq) + '\n')

    os.remove(db_fn)

def getntcontent(fasta, outfile):
    #Get nt content of sequences in fasta
    with open(fasta, 'r') as infh, open(outfile, 'w') as outfh:
        outfh.write(('\t').join(['seq', 'length', 'A', 'U', 'G', 'C']) + '\n')
        for record in SeqIO.parse(fasta, 'fasta'):
            seqid = str(record.id)
            seq = str(record.seq)
            a = round(seq.count('A') / len(seq), 4)
            c = round(seq.count('C') / len(seq), 4)
            u = round(seq.count('T') / len(seq), 4)
            g = round(seq.count('G') / len(seq), 4)
            length = str(len(seq))
            outfh.write(('\t').join([seqid, length, str(a), str(u), str(g), str(c)]) + '\n')

def getdintcontent(fasta, outfile):
    #Get dint content of sequences in fasta
    #Get all dinucleotides
    bases = 'ATCG'
    dints = []
    for dint in product(bases, repeat = 3):
        dints.append(''.join(dint))
    dintwithu = [x.replace('T', 'U') for x in dints]
    with open(fasta, 'r') as infh, open(outfile, 'w') as outfh:
        outfh.write(('\t').join(['seq', 'length'] + dintwithu) + '\n')
        for record in SeqIO.parse(fasta, 'fasta'):
            dintfreqs = []
            seqid = str(record.id)
            genename = seqid.split('|')[1]
            if genename == 'Cplx2':
                continue
            seq = str(record.seq)
            length = str(len(seq))
            for dint in dints:
                count = seq.count(dint)
                possibledint = len(seq) - 2
                freq = round(count / possibledint, 4)
                dintfreqs.append(freq)
            dintfreqs = [str(x) for x in dintfreqs]
            outfh.write(('\t').join([seqid, length] + dintfreqs) + '\n')

def makeoligoclassfastas(deseqtable, oligofasta):
    #Given a DESeq output table and a fasta file of all oligo sequences,
    #output 3 fastas, one of neurite oligos, one of soma oligos, and one of soma oligos

    oligocategories = {} #{oligoID : neurite/soma/nonenriched}
    with open(deseqtable, 'r') as infh:
        for line in infh:
            line = line.strip().split('\t')
            if line[0] == 'ensid':
                #skip header
                continue
            #Fix oligo ID so it matches with fasta
            oligoID = line[0] + '|' + line[1]
            sig = line[8]
            if sig == 'neurite':
                oligocategories[oligoID] = 'neurite'
            elif sig == 'soma':
                oligocategories[oligoID] = 'soma'
            elif sig == 'no':
                oligocategories[oligoID] = 'nonenriched'

    with open(oligofasta, 'r') as fastafh, open('neuriteoligos.fa', 'w') as neuritefh, open('somaoligos.fa', 'w') as somafh, open('nonenrichedoligos.fa', 'w') as nonenrichedfh:
        for record in SeqIO.parse(oligofasta, 'fasta'):
            try:
                oligocategory = oligocategories[str(record.id)]
            except KeyError:
                #not all oligos are in the deseq output table
                continue
            if oligocategory == 'neurite':
                neuritefh.write('>' + str(record.id) + '\n' + str(record.seq) + '\n')
            elif oligocategory == 'soma':
                somafh.write('>' + str(record.id) + '\n' + str(record.seq) + '\n')
            elif oligocategory == 'nonenriched':
                nonenrichedfh.write('>' + str(record.id) + '\n' + str(record.seq) + '\n')

def oligophastconsmean(gff, phastconsbed, outfile):
    #Get the mean phastcons score for all nt in an oligo
    gff_fn = gff
    db_fn = os.path.basename(gff_fn) + '.db'

    if os.path.isfile(db_fn) == False:
        gffutils.create_db(gff_fn, db_fn)

    db = gffutils.FeatureDB(db_fn)

    phastconsmeans = {} # {oligoID : mean phastcons score}
    interrogated_oligos = 0
    interrogated_bps = 0
    oligos_with_scores = 0
    bps_with_scores = 0
    phastconstabix = pysam.Tabixfile(phastconsbed)
    oligos = db.features_of_type('oligo')

    for oligo in oligos:
        oligolength = 0
        phastconsscores = []
        interrogated_oligos +=1
        if interrogated_oligos % 100 == 0:
            sys.stderr.write('Interrogating oligo {0}\n'.format(interrogated_oligos))

        if oligo.attributes['oligo_type'][0] == 'regular_oneexon':
            oligolength += int(oligo.stop - oligo.start)
            for bed in phastconstabix.fetch(str(oligo.chrom), oligo.start, oligo.end, parser = pysam.asTuple()):
                score = bed[4]
                phastconsscores.append(float(score))
        elif oligo.attributes['oligo_type'][0] == 'junction':
            for junctionpiece in db.children(oligo, featuretype = 'junctionpiece', level = 1):
                oligolength += int(junctionpiece.stop - junctionpiece.start)
                for bed in phastconstabix.fetch(str(junctionpiece.chrom), junctionpiece.start, junctionpiece.end, parser = pysam.asTuple()):
                    score = bed[4]
                    phastconsscores.append(float(score))

        if len(phastconsscores) > 0:
            oligos_with_scores +=1
            phastconsmeans[str(oligo.id)] = (sum(phastconsscores) / float(len(phastconsscores)))

        scored_bps = len(phastconsscores)
        interrogated_bps += oligolength
        bps_with_scores += scored_bps

    percent_bp_with_scores = (bps_with_scores / float(interrogated_bps))*100

    sys.stderr.write('Searched {0} oligos and got PhastCons scores for {1} of them. Overall, scores were obtained for {2}% of all interrogated bps.\n'.format(interrogated_oligos, oligos_with_scores, percent_bp_with_scores))

    os.remove(db_fn)

    with open(outfile, 'w') as outfh:
        for oligo in phastconsmeans:
            outfh.write(('\t').join([str(oligo), str(phastconsmeans[oligo])]) + '\n')

def getAGwindows_oligos(oligogff, genomefasta, outfile):
    #Get smoothed windows of A/G content across UTRs
    #This one is meant to take in an oligogff

    gff_fn = oligogff
    db_fn = os.path.basename(gff_fn) + '.db'

    if os.path.isfile(db_fn) == False:
        gffutils.create_db(gff_fn, db_fn)

    db = gffutils.FeatureDB(db_fn)
    utrs = db.features_of_type('UTR')

    with gzip.open(genomefasta, 'rt') as genomefh:
        print('Indexing genome...')
        seq_dict = SeqIO.to_dict(SeqIO.parse(genomefh, 'fasta'))
        print('Done!')

    with open(outfile, 'w') as outfh:
        outfh.write(('\t').join(['genename', 'position', 'agfrac']) + '\n')
        for utr in utrs:
            chrm = str(utr.chrom)
            strand = utr.strand
            genename = utr.attributes['gene_name'][0]
            for nt in range(utr.start, utr.end):
                #20 nt window
                windowstart = nt - 10
                windowend = nt + 10
                if strand == '+':
                    seq = str(seq_dict[chrm].seq[windowstart:windowend + 1]).upper()
                elif strand == '-':
                    seq = str(seq_dict[chrm].seq[windowstart:windowend + 1].reverse_complement()).upper()
                agfrac = round((seq.count('A') + seq.count('G')) / len(seq), 4)
                outfh.write(('\t').join([genename, str(nt), str(agfrac)]) + '\n')

def getAGwindows_allUTRs(UTRgff, genomefasta, outfile):
    #Get smoothed windows of A/G content across UTRs
    #This one is meant to take in a longest UTRs gff

    gff_fn = UTRgff
    db_fn = os.path.basename(gff_fn) + '.db'

    if os.path.isfile(db_fn) == False:
        gffutils.create_db(gff_fn, db_fn, merge_strategy = 'merge')

    db = gffutils.FeatureDB(db_fn)
    utrs = db.features_of_type('UTR3')

    with gzip.open(genomefasta, 'rt') as genomefh:
        print('Indexing genome...')
        seq_dict = SeqIO.to_dict(SeqIO.parse(genomefh, 'fasta'))
        print('Done!')

    with open(outfile, 'w') as outfh:
        outfh.write(('\t').join(['genename', 'chrm', 'startcoord', 'endcoord', 'metaposition', 'agfrac']) + '\n')
        for utr in utrs:
            chrm = str(utr.chrom)
            strand = utr.strand
            genename = utr.attributes['Parent'][0][5:].split('.')[0]
            if strand == '+':
                exoniccoords = []
                seq = ''
                #Get exonic coords and seqs
                for exon in db.children(utr, featuretype = 'exon', order_by = 'start'):
                    exoniccoords += list(range(exon.start, exon.end + 1))
                    exonseq = seq_dict[chrm].seq[exon.start : exon.end + 1].upper()
                    seq += exonseq

            elif strand == '-':
                exoniccoords = []
                seq = ''
                #Get exonic coords and seqs
                for exon in db.children(utr, featuretype = 'exon', order_by = 'start', reverse = True):
                    exoniccoords += list(reversed(list(range(exon.start, exon.end + 1))))
                    exonseq = seq_dict[chrm].seq[exon.start : exon.end + 1].reverse_complement().upper()
                    seq += exonseq

            if len(seq) < 100:
                continue

            for x in list(range(len(seq))):
                if x % 5 != 0:
                    continue
                #window size = 100
                #step size = 5
                if x < 50:
                    windowboundaries = [0, x + 50]
                elif len(seq) - x < 50:
                    windowboundaries = [x, len(seq)]
                else:
                    windowboundaries = [x - 50 , x + 50]

                startcoord = exoniccoords[windowboundaries[0]]
                endcoord = exoniccoords[windowboundaries[1] - 1]
                metaposition = round(x / float(len(seq)), 3)
                windowseq = seq[windowboundaries[0] : windowboundaries[1]]
                agfrac = (windowseq.count('A') + windowseq.count('G')) / len(windowseq)
                outfh.write(('\t').join([genename, chrm, str(startcoord), str(endcoord), str(metaposition), str(agfrac)]) + '\n')

'''
#Relate oligo coords to genomecoords
#needs probegff
oligocoorddf = simpleroligos2genome(sys.argv[1])
print(oligocoorddf.head())

#Define enriched windows of oligos
enrichedwindows, df = definewindows(sys.argv[2], 10)
#print(df.head())
print(enrichedwindows)

#Join oligocoord and window dfs
df = pd.merge(oligocoorddf, df, how = 'inner', on = ['genename', 'oligopos'])
print(enrichedwindows)
df.to_csv(path_or_buf = os.path.abspath(sys.argv[2]) + '.sigoligowindows', sep = '\t', na_rep = 'NA', index = False, header = True)

#Define seqs of interest based on enriched windows
minimalseqs, windowoligosgff = getminimalseqs(enrichedwindows, sys.argv[1], 0)
with open('minimalseqs.gff', 'w') as outfh:
    for minimalseq in minimalseqs:
        print(minimalseq[8], int(minimalseq[4]) - int(minimalseq[3]))
        outfh.write(('\t').join(minimalseq) + '\n')

with open('windowoligos.gff', 'w') as outfh:
    for windowoligo in windowoligosgff:
        outfh.write(('\t').join(windowoligo) + '\n')

#Make a gff for UTR regions NOT in a minimal seq
#oligogff, minimalseqgff
makenonwindowgff(sys.argv[1], 'minimalseqs.gff')

#Write fastas for minimalseq and nonminimalseq gffs
gff2seq('minimalseqs.gff', 'minimalseq', sys.argv[3], 'minimalseqs.fa')
gff2seq('nonminimalseqs.gff', 'nonminimalseqchunk', sys.argv[3], 'nonminimalseqs.fa')
'''

#getntcontent('minimalseqs.fa', 'minimalseqntcontent.txt')
#getntcontent('nonminimalseqs.fa', 'nonminimalseqntcontent.txt')
#getntcontent('windowseqs.fa', 'windowseqntcontent.txt')
#getntcontent('nonwindowseqs.fa', 'nonwindowseqntcontent.txt')
#getntcontent('neuriteoligos.fa', 'neuriteoligontcontent.txt')
#getntcontent('somaoligos.fa', 'somaoligontcontent.txt')
#getntcontent('nonenrichedoligos.fa', 'nonenrichedoligontcontent.txt')

#getdintcontent('neuriteoligos.fa', 'neuriteoligotrintcontent.nocplx2.txt')
#getdintcontent('somaoligos.fa', 'somaoligotrintcontent.nocplx2.txt')
#getdintcontent('nonenrichedoligos.fa', 'nonenrichedoligotrintcontent.nocplx2.txt')
'''
#Write fastas of oligo sequences
makeoligoclassfastas(sys.argv[1], sys.argv[2])

#Get oligo conservation
#oligogff, phastconsbed, outfile
oligophastconsmean(sys.argv[1], sys.argv[2], sys.argv[3])

'''
#Get AG content across UTRs
#oligogff, genomefasta, outfile
#getAGwindows(sys.argv[1], sys.argv[2], sys.argv[3])

getAGwindows_allUTRs(sys.argv[1], sys.argv[2], sys.argv[3])

#TODO
#generate random regions from non-minimal seq gff
#sliding G/A content window across UTR -- for each nt in the UTR (as defined by oligos.gff), calculate sliding average GAcontent


import asyncio
import logging

import aiohttp.web

async def handler(path, request):
    logging.debug('New connection received')
    try:
        ws = aiohttp.web.WebSocketResponse()
        reader, writer = await asyncio.open_unix_connection(path)
        await ws.prepare(request)
        await streamer(ws, reader, writer)
        return ws
    finally:
        logging.debug('Connection closed')

async def streamer(
    ws,
    reader: asyncio.StreamReader,
    writer: asyncio.StreamWriter,
):
    create_task = asyncio.get_event_loop().create_task
    other = create_task(tcp2ws(ws, reader))

    def ws_close(*args, **kwargs):
        create_task(ws.close())
    other.add_done_callback(ws_close)

    try:
        await ws2tcp(ws, writer)
    finally:
        other.cancel()
        await other

async def ws2tcp(
    ws,
    writer: asyncio.StreamWriter,
):
    try:
        async for data in ws:
            if data.type == aiohttp.WSMsgType.BINARY:
                writer.write(data.data)
            else:
                raise RuntimeError(f'{data.type} not supported')
    finally:
        await writer.drain()
        writer.close()
        await writer.wait_closed()

async def tcp2ws(
    ws,
    reader: asyncio.StreamReader,
):
    while True:
        data = await reader.read(1024)
        if not data:
            break
        await ws.send_bytes(data)


import footprintcore as fc

class FP_th2pad(fc.Footprint):
    kwspecs = {
        'desc':     fc.KWSpec(None, True, False),
        'spacing':  fc.KWSpec('mil', True, False),
        'drill':    fc.KWSpec('inch', True, False),
        'dia' :     fc.KWSpec('mil', False, False),
        'annulus': fc.KWSpec('mil', False, False),
        'artwidth': fc.KWSpec('mil', False, False),
        'artlen' :  fc.KWSpec('mil', False, False),
    }
    @classmethod
    def helptext(cls):
        yield "Through-hole, two-pad components."
        yield "th2pad desc=<s> dia=<mils> spacing=<mils> drill=<size, inches> artwidth=<mils> artlen=<mils>"
        yield "th2pad desc=<s> annulus=[<mils>|lhsa|shsa|asa] spacing=<mils> drill=<size, inches> artwidth=<mils> artlen=<mils>"
        yield "  desc is a description string."
        yield "  One of dia, pad diameter in mils, or annulus must be specified."
        yield "  Annulus is size in mils over effective pre-plated hole size,"
        yield "  after applying drill rack, if any is in force."
        yield "  Annulus may be specified in rules:"
        yield "    lhsa - large hand solder annulus."
        yield "    shsa - small hand solder annulus."
        yield "    asa - automatic assembly solder annulus."
        yield "  Spacing is the distance in mils between the two pad drill centers."
        yield "  Drill size is specified in fractional inches.  If a drill rack"
        yield "  is in force, the drill size will select a drill from the rack,"
        yield "  otherwise the given drill size will be used.  "
        yield "  When annulus is specified, the final pad diameter is driven by"
        yield "  the drill size chosen from the rack."
        yield "  An optional silk screen rectangle is generated by setting artwidth"
        yield "  to the width of the rectangle.  Length can be automatcially "
        yield "  calculated, or can be specified in artlen."
        yield "  Rules referenced: maskrelief, minspace, silkwidth, refdessize."
    @classmethod
    def parse(cls, footprintname, params, rules, rack, warning_callback):
        # Call the standard parameter parser.
        kw = cls.parse_kwargs(params, cls.kwspecs)
        # Compute sizes.
        drill = rack[kw['drill']]
        if 'dia' in kw:
            diameter = kw['dia']
        elif 'annulus' in kw:
            diameter = drill + 2.0 * rules.symb(kw['annulus'])
        else:
            raise fc.ParamSyntax("Must specify one of 'dia' or 'annulus'.")
        maskrelief = rules['maskrelief']
        clearance = rules['minspace']
        # Build the pin geometry
        pg = cls.thruPin.circle(drill, clearance, diameter, maskrelief)
        # Make pins
        halfwidth = kw['spacing'] / 2.0
        zero = halfwidth.u0
        pin1 = cls.pinSpec(fc.Pt(-halfwidth, zero), 1, pg)
        pin2 = cls.pinSpec(fc.Pt( halfwidth, zero), 2, pg)
        # Make silk
        box = []
        silkw = rules['minsilk']
        try:
            awhalf = kw['artwidth']/2.0
            try:
                alhalf = kw['artlen']/2.0
            except KeyError:
                alhalf = halfwidth - (diameter + maskrelief + 1.5*silkw)
            # Draw a box.
            box.append(cls.silkLine(fc.Pt( alhalf,  awhalf), fc.Pt(-alhalf,  awhalf), silkw))
            box.append(cls.silkLine(fc.Pt( alhalf, -awhalf), fc.Pt(-alhalf, -awhalf), silkw))
            box.append(cls.silkLine(fc.Pt( alhalf,  awhalf), fc.Pt( alhalf, -awhalf), silkw))
            box.append(cls.silkLine(fc.Pt(-alhalf,  awhalf), fc.Pt(-alhalf, -awhalf), silkw))
        except KeyError:
            # No silk at all.
            awhalf = fc.Dim.MIL(0) # For placing refdes
        # Make refdes
        silky = awhalf+20
        rd = cls.refDes(fc.Pt(silky.u0,silky),0, rules['minsilk'], '', rules['refdessize'])
        # Make comments
        cmt = cls.standard_comments('th2pad', kw, rules,
            ['maskrelief','minspace','minsilk','refdessize'])
        # Create the footprint instance.
        desc = str(kw['desc'])
        return cls(footprintname, desc, rd, [pin1, pin2], box, cmt)


import pygame
from pygame import draw
from minijuegos.gameobject import GameObject
from minijuegos.constantes import configuration
class Square(GameObject):
    _MAX_SPEED = 1
    def __init__(self, color = (0,0,0), pos = (0,0), width = 25, height = 25):
        self.__color = color
        self.__pos = pos
        self.__width = width
        self.__height = height
        self.__speed_x = Square._MAX_SPEED
        self.__speed_y = 0
        self.collider = pygame.Rect(0,0,0,0)

    def get_pos(self):
        return self.__pos
    def set_pos(self, x = 0, y = 0):
        self.__pos = (x,y)

    def get_speed(self):
        return (self.__speed_x, self.__speed_y)

    def set_speed(self, speed_x : int, speed_y : int):
        self.__speed_x = speed_x
        self.__speed_y = speed_y

    def update(self):
        self.__pos = (self.__pos[0] + self.__speed_x, self.__pos[1] + self.__speed_y )

    def draw(self, screen):
        result = self.out_screen()
        self.collider = draw.rect(screen,self.__color,(self.__pos[0],self.__pos[1],self.__width,self.__height))
        return result

    def out_screen(self):
        out = False
        if self.__pos[0]<0:
            out = True

        if self.__pos[0]>configuration.SCREEN_WIDTH - 25:
            out = True

        if self.__pos[1]<0:
            out = True

        if self.__pos[1]>configuration.SCREEN_HEIGHT - 25:
            out = True
        return out


from __future__ import absolute_import
import warnings

from scorify.errors import HaystackError

class Datafile(object):
    def __init__(self, lines, layout_section, rename_section):
        self.lines = lines
        self.layout_section = layout_section
        self.rename_section = rename_section
        self.header = []
        self.keep = []
        self.data = []
        super(Datafile, self).__init__()

    def read(self):
        self.header = []
        self.data = []
        self.keep = []
        for line_num, line in enumerate(self.lines):
            # Since we assume layout_section is valid, we only care about
            # header and skip and keep lines -- everything else must be data.
            line_type = ''
            if line_num < len(self.layout_section.directives):
                line_type = self.layout_section.directives[line_num].info
            if line_type == 'skip':
                continue
            if line_type == 'header':
                self.header = [
                    self.rename_section.map_name(h.strip()) for h in line]
                # Warn if header contains duplicates
                # Don't warn if there's a duplicate blank header, which we see from Excel input sometimes
                seen = set()
                duplicates = set(x for x in self.header if x in seen or (x != '' and seen.add(x)))
                if len(duplicates) > 0:
                    warnings.warn(f"Duplicates {duplicates} in header {self.header}", UserWarning)
            elif line_type == 'keep':
                self.append_keep(line)
            else:
                self.append_data(line)

    def pad_data(self, data):
        # Force lines of funny length to be the header's length
        len_diff = len(self.header) - len(data)
        padding = [''] * len_diff
        return data + padding

    def append_data(self, data):
        full_line = self.pad_data(data)
        self.data.append(dict(zip(self.header, full_line)))

    def append_keep(self, data):
        full_line = self.pad_data(data)
        self.keep.append(dict(zip(self.header, full_line)))

    def apply_exclusions(self, exclusion_section):
        new_data = []
        for row in self.data:
            try:
                exclude = any([e.excludes(row) for e in exclusion_section])
            except KeyError as exc:
                raise ExclusionError(
                    "data columns", str(exc), self.data.header)
            if not exclude:
                new_data.append(row)
        self.data = new_data

    def __len__(self):
        return len(self.data)

    def __iter__(self):
        return iter(self.data)

    def __getitem__(self, item):
        return self.data[item]

class ExclusionError(HaystackError):
    pass


from dswont.dbpedia import to_title, to_category_uri
from dswont import util
from collections import defaultdict
import re

def is_acm_id(str):
    return re.match('^\d{8}$', str) is not None

class AcmWrapper(object):

    def _read_concepts(self, filename):
        concepts = {}
        with open(filename) as f:
            for line in f.readlines():
                id, name = line.strip().split('\t')
                concepts[id] = name
        return concepts

    def _read_mapping(self, filename):
        pairs = []
        with open(filename) as f:
            for line in f.readlines():
                acm_id, wiki_title = line.strip().split('\t')
                pairs.append((acm_id, to_category_uri(wiki_title)))
        return zip(*pairs)

    def _read_relations(self, filename):
        children = defaultdict(set)
        with open(filename) as f:
            for line in f.readlines():
                parent_id, child_id = line.strip().split('\t')
                children[parent_id].add(child_id)
        return children

    def __init__(self,
                 acm_concept_file=util.resource("acm-concepts.txt"),
                 acm_rels_file=util.resource("acm-relations.txt"),
                 acm_mapping_file=util.resource("acm-wiki-mapping.txt")):

        self._concepts = self._read_concepts(acm_concept_file)

        acm_ids, wiki_uris = self._read_mapping(acm_mapping_file)
        self._wiki2acm = dict(zip(wiki_uris, acm_ids))
        self._acm2wiki = {v:k for k, v in self._wiki2acm.items()}

        self._children = self._read_relations(acm_rels_file)

    def contains(self, node):
        if is_acm_id(node):
            return node in self._acm2wiki
        else:
            return to_category_uri(node) in self._wiki2acm

    def _to_wiki(self, node):
        return self._acm2wiki[node]

    def _to_acm(self, node):
        return self._wiki2acm[to_category_uri(node)]

    def children(self, node):
        if self.contains(node):
            acm_id = self._to_acm(node)
            children = self._children[acm_id]
            return [self._to_wiki(child)
                    for child in children
                    if self.contains(child)]
        else:
            return set()

    def is_parent(self, parent, child):
        return to_category_uri(child) in self.children(parent)

    def is_ancestor(self, parent, child):
        if self.is_parent(parent, child):
            return True
        return any([self.is_ancestor(parents_child, child)
                    for parents_child in self.children(parent)])

acm = AcmWrapper()

assert acm._acm2wiki['10002950'] == to_category_uri('Mathematics of computing')
assert acm.is_parent('World Wide Web', 'Web applications')
assert not acm.is_parent('Web applications', 'Social networks')
assert not acm.is_ancestor('World Wide Web', 'Social networks')

# import pprint
# pprint.pprint(
#     [(to_title(acm._acm2wiki[parent]), to_title(acm._acm2wiki[child]))
#      for parent in acm._children.keys()
#      for child in acm._children[parent]
#      if parent in acm._acm2wiki and child in acm._acm2wiki])


# -*- coding: utf-8 -*-
# python3
import os
import hashlib
import xml.etree.ElementTree as ET
from flask import Flask, request, render_template
from handles import GetHandle, PostHandle
from db import database
from models import Model

import setting

root_path = setting.root_path
app = Flask(__name__, root_path=root_path)

# database.init_db()

basedir = os.path.abspath(os.path.dirname(__file__))

# app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + os.path.join(basedir, 'data.sqlite')
# app.config['SQLALCHEMY_COMMIT_ON_TEARDOWN'] = True
# app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = True

@app.teardown_request
def shutdown_session(exception=None):
	database.db_session.remove()

@app.route('/auth', methods=['GET', 'POST'])
def auth():
	assert request.path == '/auth', "应该路由 /auth"

	# 处理微信后台的认证请求
	if request.method == 'GET':
		return GetHandle.handle(request)
	elif request.method == 'POST':
		return PostHandle.handle(request)
	else:
		raise Exception('无法处理的request 方法: ', request.method)

@app.route('/getTodayCourse')
def get_today_course():
	from handles import PostHandle

	return PostHandle(request)

@app.route('/getTomCourse')
def get_tom_course():
	pass

app.debug = setting.debug_mode

if __name__ == '__main__':
	app.run('0.0.0.0', '80')


import datetime
import hashlib

import app
import app.model
import settings

def delete_token(ip_address, stoken):
    token = app.model.Form()
    token.token = stoken
    token.created_by_ip = ip_address

    try:
        token.delete()
        result = True
    except:
        result = False

    return result

def has_valid_token(ip_address, token):
    tokens = app.model.Form.all()
    tokens.filter("token =", token)
    tokens.filter("created_by_ip =", ip_address)
    tokens.filter("expired_at >", datetime.datetime.now())
    return tokens.get() != None

def put_form_token(ip_address, token=""):
    if token == "":
        token = make_token()

    ftoken = app.model.Form()
    ftoken.token = token
    ftoken.created_at = datetime.datetime.now()
    ftoken.created_by_ip = ip_address
    ftoken.expired_at = datetime.datetime.now() + settings.PASTE_FORM_EXPIRATION_DELTA
    key = ftoken.put()
    if key != "":
        return token
    else:
        return ""

def make_token():
    hash = hashlib.sha256()
    hash.update(datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:BONOBO"))
    return hash.hexdigest()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models
import modelcluster.fields

class Migration(migrations.Migration):

    dependencies = [
        ('wagtailcore', '0032_add_bulk_delete_page_permission'),
        ('home', '0003_add_events_block'),
    ]

    operations = [
        migrations.CreateModel(
            name='MenuItem',
            fields=[
                ('id', models.AutoField(serialize=False, verbose_name='ID', auto_created=True, primary_key=True)),
                ('title', models.CharField(max_length=255)),
            ],
        ),
        migrations.CreateModel(
            name='NavigationMenu',
            fields=[
                ('id', models.AutoField(serialize=False, verbose_name='ID', auto_created=True, primary_key=True)),
                ('title', models.CharField(max_length=255)),
            ],
            options={
                'abstract': False,
            },
        ),
        migrations.CreateModel(
            name='NavigationMenuItem',
            fields=[
                ('menuitem_ptr', models.OneToOneField(serialize=False, primary_key=True, auto_created=True, to='home.MenuItem', parent_link=True, on_delete=models.CASCADE)),
                ('sort_order', models.IntegerField(blank=True, editable=False, null=True)),
                ('parent', modelcluster.fields.ParentalKey(to='home.NavigationMenu', related_name='items')),
            ],
            options={
                'ordering': ['sort_order'],
                'abstract': False,
            },
            bases=('home.menuitem', models.Model),
        ),
        migrations.AddField(
            model_name='menuitem',
            name='link_page',
            field=models.ForeignKey(to='wagtailcore.Page', on_delete=models.CASCADE),
        ),
    ]


import pygame
import time
import math
from utils import scale_img, blit_rotate_center, blit_text_center
pygame.font.init()

# variable of the track
GRASS = scale_img(pygame.image.load('img/grass.jpg'), 2.5)
TRACK = scale_img(pygame.image.load('img/track.png'), 0.9)
HEDGE = scale_img(pygame.image.load('img/track-border.png'), 0.9)
HEDGE_MASK = pygame.mask.from_surface(HEDGE)
FINISH = pygame.image.load('img/finish.png')
FINISH_MASK = pygame.mask.from_surface(FINISH)
FINISH_POS = (130, 250)

# variable of the car
BARR = scale_img(pygame.image.load('pilots/barr.png'), 0.5)
FITT = scale_img(pygame.image.load('pilots/fitt.png'), 0.5)
MASS = scale_img(pygame.image.load('pilots/mass.png'), 0.5)
PIQ = scale_img(pygame.image.load('pilots/piq.png'), 0.5)
SEN = scale_img(pygame.image.load('pilots/sen.png'), 0.5)

# variable of the window
WIDTH, HEIGHT = TRACK.get_width(), TRACK.get_height()
WIN = pygame.display.set_mode((WIDTH, HEIGHT))
pygame.display.set_caption('Brasil Racing')
MAIN_FONT = pygame.font.SysFont("comicsans", 44)
FPS = 60
# out from 2: (110, 70), old 6: (404, 680), old 12: (734, 399),
PATH = [(175, 119), (56, 133), (70, 481), (318, 731), (418, 521), (507, 475), (600, 551), (613, 715), (736, 713), (611, 357), (409, 343), (433, 257), (697, 258), (738, 123), (581, 71), (303, 78), (275, 377), (176, 388), (178, 260)]
# laps = 2

# level class
class GameInfo:
    LEVELS = 10

    def __init__(self, level=1):
        self.level = level
        self.started = False
        self.level_start_time = 0

    def next_level(self):
        self.level += 1
        self.started = False

    def reset(self):
        self.level = 1
        self.started = False
        self.level_start_time = 0

    def game_finished(self):
        return self.level > self.LEVELS

    def start_level(self):
        self.started = True
        self.level_start_time = time.time()

    def get_level_time(self):
        if not self.started:
            return 0
        return round(time.time() - self.level_start_time)

# draw the pilots
class AbstractCar: # not to be initiate directly
    def __init__(self, max_vel, rotation_vel):
        self.img = self.IMG
        self.max_vel = max_vel
        self.rotation_vel = rotation_vel
        self.vel = 0
        self.angle = 0
        self.x, self.y = self.START_POS
        self.acc = 0.5

    def rotate(self, left=False, right=False):
        if left:
            self.angle += self.rotation_vel
        elif right:
            self.angle -= self.rotation_vel

    def draw(self, win):
        blit_rotate_center(win, self.img, (self.x, self.y), self.angle)

    def move_forward(self):
        self.vel = min(self.vel + self.acc, self.max_vel)
        self.move()

    def move_backward(self):
        self.vel = max(self.vel - self.acc, -self.max_vel/2)
        self.move()

    def move(self):
        radians = math.radians(self.angle)
        vertical = math.cos(radians) * self.vel
        horizontal = math.sin(radians) * self.vel

        self.y -= vertical
        self.x -= horizontal

    def collide(self, mask, x=0, y=0):
        car_mask = pygame.mask.from_surface(self.img)
        offset = (int(self.x - x), int(self.y - y))
        poi = mask.overlap(car_mask, offset)
        return poi

    def reset(self):
        self.x, self.y = self.START_POS
        self.angle = 0
        self.vel = 0

# player class
class Player(AbstractCar):
    IMG = SEN
    START_POS = (180, 200)

    def reduce_speed(self):
        # self.acc = 0
        self.vel = max(self.vel - self.acc/2, 0)
        self.move()

    def bounce(self):
        self.vel = -self.vel/2
        self.move()

# computer class
class Computer(AbstractCar):
    IMG = PIQ
    START_POS = (150, 200)

    def __init__(self, max_vel, rotation_vel, path=[]):
        super().__init__(max_vel, rotation_vel)
        self.path = path
        self.current_point = 0
        self.vel = max_vel

    def draw_points(self, win):
        for point in self.path:
            pygame.draw.circle(win, (255, 0, 0), point, 5)

    def draw(self, win):
        super().draw(win)
        # self.draw_points(win)

    def calculate_angle(self):
        target_x, target_y = self.path[self.current_point]
        x_diff = target_x - self.x
        y_diff = target_y - self.y

        if y_diff == 0:
            desired_radian_angle = math.pi/2
        else:
            desired_radian_angle = math.atan(x_diff/y_diff)

        if target_y > self.y:
            desired_radian_angle += math.pi

        difference_in_angle = self.angle - math.degrees(desired_radian_angle)
        if difference_in_angle >= 180:
            difference_in_angle -= 360

        if difference_in_angle > 0:
            self.angle -= min(self.rotation_vel, abs(difference_in_angle))
        else:
            self.angle += min(self.rotation_vel, abs(difference_in_angle))

    def update_path_point(self):
        target = self.path[self.current_point]
        rect = pygame.Rect(self.x, self.y, self.img.get_width(), self.img.get_height())

        if rect.collidepoint(*target):
           self.current_point += 1
            # if self.current_point == len(self.path):
              #  self.current_point = 0
    def move(self):
        if self.current_point >= len(self.path):
            return

        self.calculate_angle()
        self.update_path_point()
        super().move()

    def next_level(self, level):
        self.reset()
        self.vel = self.max_vel + (level - 1) * 0.2
        self.current_point = 0

# draw the objects
def draw(win, images, player, computer, game_info):
    for img, pos in images:
        win.blit(img, pos)

    level_text = MAIN_FONT.render(
        f"Level {game_info.level}", 1, (255, 255, 255))
    win.blit(level_text, (10, HEIGHT - level_text.get_height() - 70))

    time_text = MAIN_FONT.render(
        f"Time: {game_info.get_level_time()}s", 1, (255, 255, 255))
    win.blit(time_text, (10, HEIGHT - time_text.get_height() - 40))

    vel_text = MAIN_FONT.render(
        f"Vel: {round(player.vel, 1)}px/s", 1, (255, 255, 255))
    win.blit(vel_text, (10, HEIGHT - vel_text.get_height() - 10))

    player.draw(win)
    computer.draw(win)
    pygame.display.update()

# move player
def move_player(player):
    keys = pygame.key.get_pressed()
    moved = False

    if keys[pygame.K_a]:
        player.rotate(left=True)
    if keys[pygame.K_d]:
        player.rotate(right=True)
    if keys[pygame.K_w]:
        moved = True
        player.move_forward()
    if keys[pygame.K_s]:
        moved = True
        player.move_backward()

    if not moved:
        player.reduce_speed()

# collision detection
def handle_collision(player_car, computer_car, game_info):
    if player_car.collide(HEDGE_MASK) != None:
        player_car.bounce()

    computer_finish_poi_collide = computer_car.collide(
        FINISH_MASK, *FINISH_POS)
    if computer_finish_poi_collide != None:
        blit_text_center(WIN, MAIN_FONT, "You lost!")
        pygame.display.update()
        pygame.time.wait(5000)
        game_info.reset()
        player_car.reset()
        computer_car.reset()

    player_finish_poi_collide = player_car.collide(
        FINISH_MASK, *FINISH_POS)
    if player_finish_poi_collide != None:
        if player_finish_poi_collide[1] == 0:
            player_car.bounce()
        else:
            game_info.next_level()
            player_car.reset()
            computer_car.next_level(game_info.level)

# define race loop
race = True
clock = pygame.time.Clock()
images = [(GRASS, (0,0)), (TRACK, (0,0)), (HEDGE, (0,0)), (FINISH, FINISH_POS), (HEDGE, (0,0))]
player = Player(5, 5)
computer = Computer(5, 5, PATH)
game_info = GameInfo()

while race:
    clock.tick(FPS) # the loop can't run faster than 60 frame per second

    # blit to show the objects on screen / putting on order of later appear
    draw(WIN, images, player, computer, game_info)
    while not game_info.started:
        blit_text_center(
            WIN, MAIN_FONT, f"Press any key to start level {game_info.level}!")
        pygame.display.update()
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit()
                break
            if event.type == pygame.KEYDOWN:
                game_info.start_level()

    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            run = False
            break

    move_player(player)
    computer.move()
    handle_collision(player, computer, game_info)

    if game_info.game_finished():
        blit_text_center(WIN, MAIN_FONT, "You won the game!")
        pygame.time.wait(5000)
        game_info.reset()
        player.reset()
        computer.reset()
pygame.quit()


from unittest import mock

import pytest

from hmt_cite_atlas.library.importers import Parser
from tests import constants

@pytest.mark.parametrize("value,expected", constants.COLUMN_VARIANTS)
def test_normalize_columns(value, expected):
    assert Parser.normalize_column(value) == expected

@pytest.mark.parametrize("urn,line", constants.CITE_PROPERTIES.items())
def test_handle_citeproperties(urn, line):
    parser = Parser("some_path", mock.MagicMock())
    parser.columns = {
        "#!citeproperties": ["Property", "Label", "Type", "Authority list"]
    }
    parser.current_block = "#!citeproperties"
    parser.handle_citeproperties(line)
    obj_urn = line.split("#")[0]
    assert obj_urn in parser.columns[urn]
    assert obj_urn in parser.index


import pytest

from tests.login.login_page import LoginPage

@pytest.mark.login
class TestLogin:
    @staticmethod
    def test_title(browser):
        page = LoginPage(browser)
        expected_title = "Google"
        actual_title = page.get_title_of_page()
        assert actual_title == expected_title, f"Title should be {expected_title}, but it's {actual_title}"


"""Credit card fraud detection models. Cross-validation
of sklearn classification models with default parameters.
Parameter optimization through Grid-Search.
Kaggle dataset: https://www.kaggle.com/mlg-ulb/creditcardfraud
The dataset contains credit cards transactions made by European cardholders
in two days in September 2013: a total of 492 frauds out of 284,807 transactions.
The positive class (frauds) account for 0.172% of all transactions.
Features:
V1, V2, … V28 - numerical features the principal components obtained with PCA
Time - the seconds elapsed between each transaction and the first transaction in the data set
Amount - the transaction amount
Target value:
Class - takes value 1 in case of fraud and 0 otherwise
Metric:
Area Under the Precision-Recall Curve (AUPRC)
"""

import os
import random
import joblib

import numpy as np
import pandas as pd

from sklearn.model_selection import StratifiedKFold, cross_val_score, GridSearchCV
from sklearn.preprocessing import RobustScaler
from sklearn.pipeline import make_pipeline
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import AdaBoostClassifier
from sklearn.svm import SVC

import warnings
warnings.filterwarnings('ignore')

PATH = '../input/creditcardfraud/creditcard.csv'

SEED = 0
N_FOLDS = 3

# ----------------------- Functions ------------------------------

def set_seed(seed=42):
    """Utility function to use for reproducibility.
    :param seed: Random seed
    :return: None
    """
    np.random.seed(seed)
    random.seed(seed)
    # tf.random.set_seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    os.environ['TF_DETERMINISTIC_OPS'] = '1'

def cv_score(model, x: pd.DataFrame, y: pd.Series, metric: str) -> float:
    """Function computed cross-validation score
    for classification model based on a specified metric.
    :param model: sklearn model instance
    :param x: Original DataFrame with features and target values
    :param y: Target labels
    :param metric: Scoring metric
    :return: Mean CV score
    """
    kf = StratifiedKFold(N_FOLDS, shuffle=True, random_state=SEED)
    score = cross_val_score(model, x, y, scoring=metric, cv=kf, n_jobs=-1)
    return score.mean()

# --------------------- Feature engineering ----------------------------

set_seed(SEED)

data = pd.read_csv(PATH)
print(f'Data shape: {data.shape}')
print(data.head())

# Feature engineering: temporal features
# Obtain hour from "Time" column, which represents
# seconds passed from the 1st transaction.
# Data set contains transactions for 2 days.
data['hour'] = data['Time'] // 3600
data['hour'] = data['hour'].apply(lambda x: x if x < 24 else x - 24)

# -------------- Cross-Validation with default parameters ------------------

# Tree-based models do not require scaling of input features.
# Other classification models benefit from data preprocessing:
# scaling makes training faster and ensures better convergence.
# Possible scalers include StandardScaler, MinMaxScaler, PowerTransformer and some others.
# For this particular problem we will use RobustScaler where needed.
# This scaler is recommended for data, which has outliers.

# sklearn classification models
model_gausnb = make_pipeline(RobustScaler(), GaussianNB())
model_aboost = AdaBoostClassifier()
model_svc = make_pipeline(RobustScaler(), SVC())

models = [
    ('Gaussian Naive Bayes', model_gausnb),
    ('Ada boost', model_aboost),
    ('SVC', model_svc),
]

# Input features and target labels
x = data.drop('Class', axis=1)
y = data['Class']

for name, model in models:
    score = cv_score(model, x, y, metric='roc_auc')
    print(f'{name} model AUC score: {score}')

# ------------------- Grid-Search for optimal parameters ---------------------

# Example 1: Improve Naive Bayes classifier

# Create a dictionary of parameters and values to search.
# Since we created a pipeline including scaler and classifier,
# we need to use parameter name with double underscore after model name.
params = {
    'gaussiannb__var_smoothing': [0.001, 0.1, 0.5, 0.9, 0.95, 0.96, 0.97, 0.98, 0.99]
}

# We will use Stratified K-Fold algorithm to train the models
# on several subsets of data and check the AUC score of validation subsets.
kf = StratifiedKFold(N_FOLDS, shuffle=True, random_state=SEED)

# Define Grid-Search
grid = GridSearchCV(model_gausnb, params, cv=kf, scoring='roc_auc',
                    n_jobs=-1, refit=True)

# Display the results
grid.fit(x, y)
print('Best AUC score:', grid.best_score_)
print(grid.best_params_)

# Save the best model.
joblib.dump(grid, 'gaussiannb.pkl')

# Example 2: Improve AdaBoost classifier

# In this case we use model without preprocessing step.
# Parameters in the dictionary are named the same as in model definition.
# Parameter space is limited to reduce grid-search time.
params = {
    'n_estimators': [50, 60],
    'learning_rate': [0.7, 0.8]
}

grid = GridSearchCV(model_aboost, params, cv=kf, scoring='roc_auc',
                    n_jobs=-1, refit=True)

grid.fit(data.drop('Class', axis=1), data['Class'])
print('Best AUC score:', grid.best_score_)
print(grid.best_params_)

joblib.dump(grid, 'adaboost.pkl')


#!/bin/bsh/

count = 0
maxcount = 0
result = 0

for i in range(len(s) - 1):
    if (s[i] <= s[i + 1]):
        count += 1
        if count > maxcount:
            maxcount = count
            result = i + 1
    else:
        count = 0
startposition = result - maxcount
print('Longest substring in alphabetical order is:', s[startposition:result + 1])


# -*- coding: cp936 -*-
import os
import shutil

def space2TabAnd2Upper(filePath_space,filePath_tab):
        fileOpened_space=open(filePath_space,'r')
        fileWrited_txtLog=open(filePath_tab,'w')
        lineIndex=0
        for line in fileOpened_space.readlines():
            lineIndex+=1
            splitLine=line.split()
            for i in range(len(splitLine)):
                splitLine[i]=splitLine[i].upper()
            fileWrited_txtLog.write('\t'.join(splitLine)+'\n')
        fileWrited_txtLog.close()
        print(filePath_space+"convert to tab complete.")

if __name__=="__main__":

    sourceDirPath="testData"
    goalDirPath='222'
    if os.path.exists(goalDirPath):
        shutil.rmtree(goalDirPath)
    os.mkdir(goalDirPath)

    ##  把操作目录下文件存入filenameslist
    fileNames=os.listdir(sourceDirPath)
    for fileItem in fileNames:
        print ('-'*10,'Current dealing...'+fileItem)
        fileOpened=sourceDirPath+'\\'+fileItem
        fileWrited=goalDirPath+'\\'+fileItem
        space2TabAnd2Upper(fileOpened,fileWrited)

    print("新文件在"+goalDirPath)

文件读写及处理/目录中所有文件中小写变大写_空格变tab.py

目录中所有文件中小写变大写_空格变tab.py


import datetime
import pathlib
import pandas as pd
import subprocess as sp
from typing import Tuple
import xml.etree.ElementTree as et

nmap_direct, mass_direct = 'Scans/Nmap', 'Scans/Masscan'

def direct_create():
    #
    directories = ['Scans', nmap_direct, mass_direct]

    for directory in directories:
        pathlib.Path(directory).mkdir(exist_ok=True)

def masscan_parser(masscan_output: str) -> Tuple[str, str]:
    """
    Takes a file produced from Masscan's -oL option, returns online IP addresses and listening ports
    :param masscan_output:
    :return addresses, ports:
    """

    with open(masscan_output) as f:
        temp_data = f.readlines()

    addresses = " ".join({host.split()[3] for host in temp_data if len(host.split(' ')) > 2})
    ports = ",".join({host.split()[2] for host in temp_data if len(host.split(' ')) > 2})

    return addresses, ports

def host_parser(raw_txt: str) -> str:
    """
    Simple function to parse a host from Nmap txt file.
    """
    start = 'nmap scan report for'
    end = 'host'
    raw_txt = raw_txt.lower()

    host = raw_txt[raw_txt.find(start) + 20:raw_txt.find(end)]

    return host

def port_parser(raw_txt: str) -> str:
    """
    Simple function to parse port information from Nmap txt file.
    """
    start = 'service'
    ports = raw_txt[raw_txt.find(start) + 7:]
    ports = [port for port in ports.split('\n') if port]
    print(ports)
    ports = [port.split() for port in ports]

    ports = [port for port in ports if len(port) == 3]

    return ports

def df_generator(raw_txt: str):
    """
    Uses host and port parsers above to spit out a semi-usable dataframe.
    """
    raw_data = raw_txt.split('\n\n\n')
    records = []

    for finding in raw_data:
        host = host_parser(finding)
        ports = port_parser(finding)

        for port in ports:
            records.append([host] + ports)

    df_scan_data = pd.DataFrame(records)

    return df_scan_data

def xml_to_df(scan_xml):
    """
    (Try) to convert Nmap XML scan output into a Pandas dataframe.
    """

    scan_data = [] # Empty list to which data will be appended.

    # Read in XML file, establish XML root, then parse all scanned hosts.
    tree = et.parse(scan_xml)
    root = tree.getroot()
    hosts = root.findall('host')

    # Iterate over hosts to find IP, Status, hostnames, ports, and associated port information.
    for host in hosts:
        address = host.findall('address')[0].attrib['addr']
        status = host.findall('status')[0].attrib['state']

        try:
            hostnames = host.findall('hostnames')[0].attrib['name']
        except Exception as e:
            hostnames = ''

        port_element = host.findall('ports')
        ports = port_element[0].findall('port')
        for port in ports:
            port_data = []
            proto = port.attrib['protocol']
            port_id = port.attrib['portid']
            try:
                service = port.findall('service')[0].attrib['name']
            except Exception as e:
                service = ''
            try:
                state = port.findall('state')[0].attrib['state']
            except Exception as e:
                state = ''

            # Create a list of the port data
            port_data.extend((address, status, hostnames,
                                proto, port_id, service, state))

                # Add the port data to the host data
            scan_data.append(port_data)

    # Establish dataframe columns and create the dataframe.
    df_cols = ['host', 'status', 'hostnames', 'protocol', 'port', 'service', 'port_state']
    df_scan = pd.DataFrame(data=scan_data, columns=df_cols)

    return df_scan

def scan_handler(ranges: dict, timestamp: datetime, port_quant: str = '1-65535', port_type: str = 'tcp_udp',
                 os_detection: bool = True):
    for site, subnet in ranges.items():
        target_mass_file = str(pathlib.Path(f"Scans/Masscan/{site}_mass_{timestamp}"))
        target_nmap_file = str(pathlib.Path(f"Scans/Nmap/{site}_{timestamp}"))

        mass_cmd = f"masscan {subnet} "
        nmap_cmd = "nmap "

        if port_type == 'tcp_udp':
            mass_cmd += f"-p{port_quant},U:{port_quant}"
            nmap_cmd += f"-sS -sU "

        elif port_type == 'tcp':
            mass_cmd += f"-p{port_quant}"
            nmap_cmd += f"-sS "

        else:
            mass_cmd += f"--udp-ports {port_quant}"
            nmap_cmd += f"-sU "

        mass_cmd += f" --rate 10000 -oL {target_mass_file}.txt"

        sp.run(mass_cmd, shell=True)

        # Load, clean, and format our masscan txt for nmap
        addresses, ports = masscan_parser(f"{target_mass_file}.txt")

        if os_detection:
            nmap_cmd += "-O "

        # Finish building our Nmap command with collected data from Masscan, then execute
        nmap_cmd += f"-p {ports} -Pn {addresses} -oX {target_nmap_file}.xml"
        sp.run(nmap_cmd, shell=True)


from manim import *
from math import sin, cos

class IntroScene(Scene):
    """
    Scene for the intro of the video.
    """

    def construct(self):
        MAX_SIDES = 8
        ref_shape = None
        for side_n in range(3, MAX_SIDES + 1):
            # Regular Polygon animation
            angle = TAU / side_n
            co_ords = [[sin(i * angle), cos(i * angle), 0]
                       for i in range(side_n)]
            polygon = Polygon(
                *co_ords).set_fill(random_bright_color(), opacity=0.5)

            if not ref_shape:
                ref_shape = polygon
                self.add(ref_shape)
            else:
                self.play(ReplacementTransform(ref_shape, polygon))
                ref_shape = polygon

        self.play(ReplacementTransform(ref_shape, Tex("Hello, World!")))
        self.wait(2)

class ProblemStatingScene(Scene):
    """
    Scene for stating the problem statement.
    """

    def construct(self):
        statement = Tex(
            r"What is the the sum of all the natural numbers?\\1+2+3+4+...+n=?")
        self.play(Write(statement))
        self.wait(3)

class RamanujanSummationScene(Scene):
    """
    Scene for explaining the ramanujan summation.
    """

    def construct(self):
        info = Tex(
            r"\justifying {A great Indian mathematician Sir Srinivasa Ramanujan calculated the sum of all the natural numbers in a very interesting way. We address the method as Ramanujan Summation.}").scale(0.75).to_corner(UP + LEFT)
        self.play(Write(info))
        self.wait(5)

        proof_a = Tex(r"\justifying {Proof:\\$A=1-1+1-1+...$\\$A=1-(1-1+1-1+...)$\\$A=1-A$\\$A=1/2$}").scale(
            0.75).next_to(info, DOWN).shift(DOWN).to_corner(LEFT)
        self.play(Write(proof_a))
        self.wait(6)

        proof_b = Tex(r"\justifying {$B=1-2+3-4+...$\\$A-B=(1-1+1-1+...)-(1-2+3-4+...)$\\$A-B=(1-1)+(-1+2)+(1-3)+(-1+4)+...$\\$A-B=1-2+3-4+...$\\$A-B=B$\\$B=1/4$}").scale(
            0.75).next_to(proof_a, RIGHT).shift(0.1 * DOWN)
        self.play(Write(proof_b))
        self.wait(6)

        self.clear()

        proof_c = Tex(r"\justifying {$S=1+2+3+4+...$\\$B-S=(1-2+3-4+...)-(1+2+3+4+...)$\\$B-S=(1-1)+(-2-2)+(3-3)+(-4-4) +...+(5-5)+(-6-6)+...$\\$B-S=-4(1+2+3+...)$\\$B-S=-4S$\\$S=-1/12$}").scale(
            0.75).to_corner(UP + LEFT)
        self.play(Write(proof_c))
        self.wait(6)

        self.clear()

        flushed_img = ImageMobject(
            "resources/flushed.png").scale(0.4).to_corner(UP)
        self.add(flushed_img)
        self.wait(1)

        exclamation = Tex(r"\justifying {sum of positive numbers = negative?}").next_to(
            flushed_img, DOWN)
        self.play(Write(exclamation))
        self.wait(1)

        end_line = Tex(r"\justifying {Sir Ramanujan wasn't right here, let's see how!}").next_to(
            exclamation, DOWN).shift(0.6*DOWN)
        self.play(Write(end_line))
        self.wait(2)

class SummationFlawScene(Scene):
    """
    Scene for explaining the flaw of ramanujan summation.
    """

    def construct(self):
        reason = Tex(
            r"\justifying {The only flaw in the ramanujan summation is that we can't treat a divergent series as a convergent one.}").scale(0.75).to_corner(UP + LEFT).set_color(RED)
        self.play(Write(reason))
        self.wait(3)

        proof = Tex(r"\justifying {Real answer to the problem:\\$S_{n}=1+2+3+4+...+n$\\$S_{n}=\frac{n(n+1)}{2}$\\$\lim\limits_{n \to \infty} \frac{n(n+1)}{2} = \infty$}").scale(0.75).next_to(
            reason, DOWN).shift(0.4 * DOWN).to_corner(LEFT).set_color(GREEN)
        self.play(Write(proof))
        self.wait(3)

        conclusion = Tex(r"\justifying {As the number of terms in the sum increases, the sum tends to infinity}").scale(0.75).next_to(
            proof, DOWN).shift(0.6 * DOWN).to_corner(LEFT)
        self.play(Write(conclusion))
        self.wait(3)

class SummationFactScene(Scene):
    """
    Scene for explaining the fact about ramanujan summation.
    """

    def construct(self):
        ax = Axes(
            x_range=[-1.8, 0.8, 1],
            y_range=[-0.5, 0.72, 1],
            tips=False,
            axis_config={"include_numbers": True},
        )

        self.play(Create(ax))
        self.wait(1)

        graph = ax.plot(lambda x: x / 2 * (x + 1),
                        x_range=[-1.8, 0.8], use_smoothing=True)
        self.play(Create(graph))
        self.wait(1)

        area = ax.get_area(graph, x_range=(-1.0, 0), color=RED, opacity=0.5)
        self.play(Create(area))
        self.wait(3)

        fact = Tex(r"\justifying {Area of red region = $-1/12$}").scale(
            0.75).next_to(area, LEFT).shift(0.3 * RIGHT + 1.1 * DOWN)
        self.play(Write(fact))
        self.wait(4)

class OutroScene(Scene):
    """
    Scene for the outro of the video.
    """

    def construct(self):
        self.play(Write(Tex(
            r"Thank You!\\And don't forget that sum of all the natural numbers isn't -1/12 ;)").scale(0.9)))
        self.wait(3)