#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Author: Leonel Figueiredo de Alencar
# Last update: June 8, 2023

from Nheengatagger import getparselist, tokenize, DASHES, ELLIPSIS
from BuildDictionary import DIR,MAPPING, extract_feats, loadGlossary, loadLexicon, extractTags, isAux, accent, guessVerb
from conllu.models import Token,TokenList
from conllu import parse
from io import open
from conllu import parse_incr
import re, os

# set with lexicalized reduplications
LEXREDUP=set()

# reduplication tag
REDUP='RED'

# default annotator's name abbreviation
ANNOTATOR = 'LFdeA'

# path to treebank file
TREEBANK_FILE='yrl_complin-ud-test.conllu'
TREEBANK_DIR='corpus/universal-dependencies'
TREEBANK_PATH=os.path.join(DIR,TREEBANK_DIR, TREEBANK_FILE)

# set with all treebank sentences
TREEBANK_SENTS=[]

# single and double quotes
QUOTES='''"''"'''

# punctuation introducing a dependent clause
# TODO: the comma needs a special treatment when introducing non-clausal dependents
COMMA=','
COLON=':'
SEMICOLON=';'
DEPPUNCT=[COMMA,COLON,SEMICOLON]
DEPPUNCT.extend(DASHES)

# characters to be removed from input sentence
REMOVE=re.compile(r"/=?[\w\+]*([:=|]\w+)*@?")

# regex defining pattern to parse examples
PARTS=re.compile(r"\s+-\s+|[)(]")

# regex matching tags like 'N+ABS', 'N+NCONT', 'ABS', 'NCONT', etc.
TAGSEQ=re.compile(r'(^\w+\+)?(\w+)\b')

# separators of multiword tokens
HYPHEN='-'
UNDERSCORE='_'

# sentence terminators
SENTTERM=('.','?','!')

# Multiword token
MULTIWORDTOKENS={}

# Case of second class pronouns
CASE="Gen"

# Enclitic postpositions
# TODO: extract fom glossary
# hyphenated
CLITICS=['pe']

# non-hyphenated
# clitic alomorph of postposition 'upé'
ME='me'

# clitic adverb "-ntu"
NTU='ntu'

# clitic question particle "-ta"
TA='taá'

NONHYPHEN=[NTU,ME]

ROOT=[]

GLOSSARY=loadGlossary(jsonformat=os.path.join(DIR,"glossary.json"))
LEXICON=loadLexicon()

UDTAGS={'PL': 'Plur', 'SG': 'Sing',
'V': 'VERB', 'N': 'NOUN', 'V2': 'VERB', 'V3': 'VERB',
'VSUFF': 'VERB',
'A': 'ADJ', 'A2': 'VERB',
'CONJ' : 'C|SCONJ', 'NFIN' : 'Inf', 'ART' : 'DET',
'COP' : 'AUX', 'PREP' : 'ADP', 'SCONJR': 'SCONJ',
'AUXN' : 'AUX', 'AUXFR' : 'AUX', 'AUXFS' : 'AUX',
'CARD' : 'NUM', 'ORD' : 'ADJ', 'ELIP' : 'PUNCT',
'COL':'Coll', 'PRV': 'Priv', 'RELF' : 'PRON', 'RED': 'Red'}

# TODO: extractDemonstratives()
DET =  {'DEM' : 'DET', 'INDQ' : 'DET',
'INT' : 'DET', 'ART' : 'DET', 'DEMX' : 'DET', 'DEMS' : 'DET',
'DEMSN' : 'DET', 'IND' : 'DET', 'TOT' : 'DET'}

DEIXIS={'DEMS' : 'Remt', 'DEMSN' : 'Remt','DEMX' : 'Prox', 'I': 'Remt', 'X': 'Prox'}

ADVTYPE={'ADVO':'Tim','ADVD':'Loc', 'ADVS':'Deg', 'ADVJ':'Cau', 'ADVG':'Deg', 'ADVP':'Con'}

WH_ADVTYPE={'ADVC':'Loc', 'ADVA':'Man',
'ADVT':'Tim', 'ADVM':'Mod', 'ADVU': 'Cau'}

WH_ADV={'ADVR': 'Int', 'ADVL':'Rel', 'ADVN': 'Ind'}
DEM_ADV={'ADVD': 'Dem'}
ADVPRONTYPE={}
ADVPRONTYPE.update(WH_ADV)
ADVPRONTYPE.update(DEM_ADV)

ADVTYPE.update(WH_ADVTYPE)

def extract_redup(glossary=GLOSSARY):
    lexredup=set()
    entries=list(filter(lambda x: HYPHEN in x['lemma'], glossary))
    for entry in entries:
        lemma=entry['lemma']
        parts=lemma.split(HYPHEN)
        if len(parts) == 2 and parts[0] == parts[1]:
            lexredup.add(lemma)
    return lexredup

LEXREDUP=extract_redup()

def extractTreebankSents(infile=TREEBANK_PATH):
    sents=extractConlluSents(infile)
    for sent in sents:
        text=sent.metadata.get('text_orig')
        if not text:
            text=sent.metadata.get('text')
        TREEBANK_SENTS.append(text)

def checkSentence(sent):
    if not TREEBANK_SENTS:
        extractTreebankSents()
    sents=list(filter(lambda x:x == sent, TREEBANK_SENTS))
    if sents:
        return True
    return False

def extractAuxiliaries(tag='aux.'):
    glossary=loadGlossary()
    auxiliaries=list(filter(lambda x: tag in x.get('pos'),glossary))
    for auxiliary in auxiliaries:
        pos=auxiliary['pos']
        auxiliary['pos']=extractTags(pos,isAux)[0]
    return auxiliaries

AUX = extractAuxiliaries()

def extractAuxEntry(lemma):
    entry=dict()
    entries=list(filter(lambda x: lemma == x.get('lemma'), AUX))
    if entries:
        entry.update(entries[0])
    return entry

def AdverbTags():
    adv='adv.'
    upos='ADV'
    dic={}
    for pos,xpos in MAPPING.items():
        if pos.startswith(adv):
            dic[xpos]=upos
    return dic

def extractAdverbs():
    advs={}
    for lemma,entries in LEXICON.items():
        for entry in entries:
            if entry[1].startswith('ADV'):
                if advs.get(lemma):
                    advs[lemma].append(entry[1])
                else:
                    advs[lemma]=[entry[1]]
    return advs

def extractParticles(mapping):
    dic={}
    for k,v in mapping.items():
        if 'part.' in k:
            dic[v]='PART'
    return dic

def extractPronouns(mapping):
    dic={}
    for k,v in mapping.items():
        if 'pron.' in k:
            ud='PRON'
            t=DET.get(v)
            if t:
                ud=t
            dic[v]=ud
    return dic

UDTAGS.update(extractParticles(MAPPING))
UDTAGS.update(extractPronouns(MAPPING))
UDTAGS.update(AdverbTags())

def tokenrange(sentence):
    m=0
    for token in sentence.split():
        if not token.endswith("."):
            n=m+len(token)
            print(token,m,n)
            m=n+1
        else:
            token=token[:-1]
            n=m+len(token)
            print(token,m,n)
            print(".",n,n+1)

def incrementRange(r='107:111',i=1):
	m,n=[int(x) for x in r.split(':')]
	return f"{m+i}:{n+i}"

def incrementTokenRange(tokenlist,i=1):
    for t in tokenlist:
        value=incrementRange(t['misc']['TokenRange'],i)
        t['misc']['TokenRange']=value

def getudtag(tag):
    udtag=UDTAGS.get(tag)
    if udtag:
        return udtag
    else:
        return tag.upper()

def RelAbbr(rel):
    if rel == 'NCONT':
        return 'NCont'
    return rel.title()

def mkTokenRange(token, start, end):
    token['misc']={'TokenRange': f'{start}:{end}'}

def mkMultiWordToken(ident,form,start=0,end=0,spaceafter=None):
    token=Token()
    token['id']=ident
    token['form']=form
    for field in ('lemma','upos','xpos','feats','head','deprel','deps'):
        token[field] = None
    mkTokenRange(token,start,end)
    if spaceafter:
        token['misc'].update({'SpaceAfter':'No'})
    return token

def WordsOfLenghth(lenght,pos):
    glossary=loadGlossary()
    return list(filter(lambda x:  len(x['lemma']) == lenght and x['pos'] == pos,glossary))

def mkConlluToken(word,entry,head=0, deprel=None, start=0, ident=1, deps=None):
    mapping={'ADP' : 'case', 'SCONJ':'mark',
    'VERB':'root',
    'PUNCT':'punct'}
    end=start + len(word)
    feats={}
    token=Token()
    token['id']=ident
    token['form']=word
    token['lemma']=entry['lemma']
    pos=entry.get('pos')
    if pos:
        token['upos']=getudtag(pos)
        token['xpos']=pos
    else:
        token['upos']=None
        token['xpos']=None
    upos=token['upos']
    person=entry.get('person')
    number=entry.get('number')
    degree=entry.get('degree')
    derivation=entry.get('derivation')
    redup=entry.get('redup')
    aspect=entry.get('aspect')
    tense=entry.get('tense')
    vform=entry.get('vform')
    mood=entry.get('mood')
    rel=entry.get('rel')
    if person:
        feats['Person']=person
        if token['upos']=='VERB':
            feats['VerbForm']='Fin'
    if number:
        feats['Number']=getudtag(number)
    if vform:
        feats['VerbForm']=getudtag(vform)
    if rel:
        feats['Rel']=RelAbbr(rel)
        handleNCont(upos,feats)
    if degree:
        feats['Degree']=f"{degree.title()}"
    if aspect:
        feats['Aspect']=f"{aspect.title()}"
    if mood:
        feats['Mood']=f"{mood.title()}"
    if tense:
        feats['Tense']=f"{tense.title()}"
    if derivation:
        feats['Derivation']=UDTAGS.get(derivation)
    if redup:
        feats['Red']='Yes'
    if feats:
        token['feats']=feats
    else:
        token['feats']=None
    if token['xpos'] in ('REL', 'RELF'):
        updateFeats(token,'PronType','Rel')
    #elif token['xpos'] == 'ADVR':
    #    updateFeats(token,'PronType','Int')
    #elif token['xpos'] == 'ADVL':
    #    updateFeats(token,'PronType','Rel')
    elif token['xpos'] in ('ORD','ADVO'):
        updateFeats(token,'NumType','Ord')
    includeAdvType(token)
    token['head']=head
    dprl=mapping.get(upos)
    if not dprl:
        dprl=deprel
    token['deprel']=dprl
    token['deps']=deps
    token['misc']={'TokenRange': f'{start}:{end}'}
    return token

def spaceBefore(token):
    if token['xpos'] == 'ELIP' or token['lemma'] in DEPPUNCT:
        return True

def insertNoSpaceAfter(token):
    if not spaceBefore(token):
        token['misc'].update({'SpaceAfter' : 'No'})

def handleSpaceAfter(tokenlist):
    i=0
    while(i<len(tokenlist)-1):
        token=tokenlist[i]
        nexttoken=tokenlist[i+1]
        if token['lemma'] == ',':
            misc=token.get('misc')
            if misc.get('SpaceAfter'):
                misc.pop('SpaceAfter')
        elif token['lemma'] == ELLIPSIS and nexttoken['lemma'] in SENTTERM+(',',):
            token['misc'].update({'SpaceAfter' : 'No'})
        i+=1
    tokens=tokenlist.filter(upos='PUNCT')
    quotes=[token for token in tokenlist if token['form'] in QUOTES]
    if tokens:
        previousHasSpaceAfter=False
        i=0
        c=len(tokenlist)
        while(i<len(tokens)):
            precedentlist=tokenlist.filter(id=tokens[i]['id']-1)
            if tokens[i] in quotes:
                if previousHasSpaceAfter:
                    handlePrecedent(precedentlist)
                    quotid=previousCat(tokens[i],quotes)
                    quote=tokens.filter(id=quotid)[0]
                    tokens[i]['head']=quote['head']
                    previousHasSpaceAfter=False
                else:
                    insertNoSpaceAfter(tokens[i])
                    tokenid=tokens[i]['id']
                    head=tokenlist.filter(id=tokenid+1)[0]
                    tokens[i]['head']=head['id']
                    previousHasSpaceAfter=True
            else:
                handlePrecedent(precedentlist)
            if tokens[i]['lemma'] in SENTTERM:
                insertNoSpaceAfter(tokens[i])
            i+=1

def handlePrecedent(precedentlist):
    for precedent in precedentlist:
        insertNoSpaceAfter(precedent)

def sortDict(d):
    l=list(d.items())
    l.sort()
    return dict(l)

def sortTokens(tokenlist):
    for token in tokenlist:
        for k,v in token.items():
            if isinstance(v, dict) and len(v) > 1:
                token[k]=sortDict(v)

def VerbIdsList(tokenlist):
    return tokenlist.filter(upos="VERB")

def VerbIdsList1(tokenlist):
    return VerbsList(tokenlist)

def isVerb(token):
    'return True if token is a verb or an existential or presentative particle'
    if token.get('upos') == 'VERB' or token.get('xpos') in ('EXST', 'PRSV'):
        return True
    return False

def VerbsList(tokenlist):
    verblist=list(filter(lambda x: isVerb(x), tokenlist))
    tk=TokenList()
    tk.extend(verblist)
    return tk

def TokensOfCatList(tokenlist,cat):
    return tokenlist.filter(upos=cat)

def FirstVerbId(tokenlist):
    verbs=VerbIdsList(tokenlist)
    if verbs:
        return verbs[0]['id']
    return 0

def getprontype(xpos):
    prontype='Prs'
    if xpos not in ('PRON', 'PRON2'):
        prontype=xpos.title()
        if len(prontype) > 3 and prontype not in ('Card',):
            prontype=prontype[:3]
    return prontype

def handlePron(token,nextToken):
    xpos=token['xpos']
    prontype=getprontype(xpos)
    if not token.get('feats'):
        token['feats']={}
    token['feats'].update({'PronType': prontype})
    if xpos == 'PRON2':
        token['feats'].update({'Case': CASE})
        if nextToken['upos'] == 'NOUN':
            token['feats'].update({'Poss': 'Yes'})
            token['deprel'] ='nmod:poss'
            token['head'] =nextToken['id']

def setDeprel(token, headid,deprel='nsubj',overwrite=False):
    if overwrite:
        token['deprel'] = deprel
        token['head'] =  headid
    else:
        if not token['deprel']:
            token['deprel'] = deprel
            token['head'] =  headid

def SubjOrObj(token,verbs):
    previousverb = previousVerb(token,verbs)
    nextverb = nextVerb(token,verbs)
    if previousverb:
        headlist=verbs.filter(id=previousverb)
        if headlist:
            head=headlist[0]
            upos=head['upos']
            if upos == 'VERB':
                setDeprel(token,previousverb,'obj')
            else:
                setDeprel(token,previousverb,'nsubj')
    elif nextverb:
        setDeprel(token,nextverb,'nsubj')

def hasNext(token,nexttoken):
    return token['id'] == nexttoken['id'] - 1

def sameHead(token,nexttoken):
    return token['head']==nexttoken['head']

def sameDeprel(token,nexttoken):
    dep1=token['deprel']
    dep2=nexttoken['deprel']
    return dep1==dep2

def ListOfCats(tokenlist,cats):
    newlist=TokenList()
    for token in tokenlist:
        if token.get('upos') in cats:
            newlist.append(token)
    return newlist

def handleNmodPoss(tokenlist):
    nouns=ListOfCats(tokenlist,('NOUN','PROPN'))
    i=0
    start=0
    while(i < len(nouns)-1):
        token=nouns[i]
        j=i+1
        nexttoken=nouns[j]
        feats=nexttoken.get('feats')
        rel=''
        if feats:
            rel=feats.get('Rel')
        if hasNext(token,nexttoken):
            if rel == 'Cont' or (sameHead(token,nexttoken) and sameDeprel(token,nexttoken)):
                start=j
                token['deprel']='nmod:poss'
                token['head']=nexttoken['id']
        i+=1
    GenitiveConstruction(nouns,tokenlist)
    return start

def handlePospCompl(token,nextToken):
    nextToken['deprel']='case'
    nextToken['head'] =token['id']

def handleAdpCompl(token,verbs,nextToken):
    token['deprel'] = 'obl'
    verb=previousVerb(token,verbs)
    if not verb:
        verb=nextVerb(token,verbs)
    if verb:
        token['head'] = verb
    posp=nextToken['xpos'] == 'ADP'
    if posp:
        handlePospCompl(token,nextToken)

def setAttribute(token,attribute,value,overwrite=False):
    if overwrite:
        token[attribute] = value
    else:
        if not token[attribute]:
            token[attribute] = value

def handleNounPron(token,nextToken,verbs):
    upos=''
    if nextToken:
        upos=nextToken['upos']
    if upos == 'ADP':
        handleAdpCompl(token,verbs,nextToken)
    elif upos == 'ADJ':
        if token['upos'] == 'NOUN':
            #nextToken['deprel']='amod'
            #nextToken['head'] =token['id']
            setAttribute(nextToken,'deprel','amod')
            if nextToken['deprel']=='amod':
                nextToken['head'] =token['id']
    elif upos == 'VERB':
        token['deprel'] = 'nsubj'
        token['head'] =nextToken['id']
    if  token['xpos'] != 'PRON2':
        SubjOrObj(token,verbs)

def updateFeats(token,feature,value):
    if not token['feats']:
        token['feats']={}
    token['feats'].update({feature : value})

def headPartNextVerb(token,verbs):
    tokid=token['id']
    for verb in verbs:
        verbid=verb['id']
        if verbid > tokid:
            token['head']=verbid
            break

def headPartPreviousVerb(token,verbs):
    tokid=token['id']
    i=len(verbs)-1
    while(i >=0):
        verb=verbs[i]
        verbid=verb['id']
        if verbid < tokid:
            token['head']=verbid
            break
        else:
            headPartNextVerb(token,verbs)
        i=i-1

def handleIntj(token,verbs):
    token['deprel'] = 'discourse'
    headPartNextVerb(token,verbs)

def PreviousContentWord(token,tokenlist):
    funct=['ADP','AUX','CONJ','SCONJ']
    return getPreviousToken(token,tokenlist,funct)

def handlePart(token,tokenlist,verbs):
    mapping= {
    'PQ': {'PartType': 'Int','QestType':'Polar'},
    'CQ': {'PartType': 'Int','QestType':'Content'},
    'NEG': {'PartType': 'Neg','Polarity':'Neg'},
    'CONS': {'PartType': 'Mod','Polarity':'Pos'},
    'NEGI': {'PartType': 'Neg','Polarity':'Neg','Mood': 'Imp'},
    'RPRT': {'PartType': 'Mod','Evident':'Nfh'},
    # 'RPRT': {'PartType': 'Tam','Evident':'Nfh'},
    'PFV': {'PartType': 'Tam','Aspect':'Perf'},
    'FRUST': {'PartType': 'Tam','Aspect':'Frus'},
    'FUT': {'PartType': 'Tam','Tense':'Fut'},
    'PRET': {'PartType': 'Tam','Tense':'Past'},
    'EXST': {'PartType': 'Exs'},
    'PRSV': {'PartType': 'Prs'},
    'CERT': {'PartType': 'Mod'},
    'ASSUM': {'PartType': 'Mod'},
    'PROTST': {'PartType': 'Mod'},
    'TOTAL': {'PartType': 'Quant', 'Aspect':'Compl'},
    'COND': {'PartType': 'Mod', 'Mood': 'Cnd'},
    'NEC': {'PartType': 'Mod', 'Mood': 'Nec'},
    'FOC': {'PartType': 'Emp', 'Foc': 'Yes'}
    }
    token['deprel'] = 'advmod'
    xpos=token['xpos']
    if xpos.startswith('NEG'): #TODO: use the dictionary instead
        updateFeats(token,'Polarity', 'Neg')
        updateFeats(token,'PartType', 'Neg')
        if xpos == 'NEGI':
            updateFeats(token,'Mood', 'Imp')
        headPartNextVerb(token,verbs)
    elif xpos == 'RPRT':
        updateFeats(token,'Evident','Nfh')
        updateFeats(token,'PartType', 'Mod')
        headPartPreviousVerb(token,verbs)
    elif xpos == 'CERT' or xpos == 'ASSUM':
        updateFeats(token,'PartType', 'Mod')
        token['head']=getAdvHead(token,tokenlist,verbs)
        # headPartPreviousVerb(token,verbs)
    elif xpos == 'NEC':
        updateFeats(token,'PartType', 'Mod')
        headPartNextVerb(token,verbs)
    elif xpos == 'PFV':
        #headPartPreviousVerb(token,verbs)
        token['head']=getAdvHead(token,tokenlist,verbs)
        updateFeats(token,'Aspect','Perf')
    elif xpos == 'TOTAL':
        #headPartPreviousVerb(token,verbs)
        token['head']=getAdvHead(token,tokenlist,verbs)
        updateFeats(token,'Aspect','Compl')
    elif xpos == 'FRUST':
        headPartPreviousVerb(token,verbs)
        updateFeats(token,'Aspect','Frus')
    elif xpos == 'PROTST':
        headPartPreviousVerb(token,verbs)
        updateFeats(token,'PartType', 'Mod')
    elif xpos == 'FUT':
        #headPartPreviousVerb(token,verbs)
        token['head']=getAdvHead(token,tokenlist,verbs)
        updateFeats(token,'Tense','Fut')
    elif xpos == 'PRET':
        headPartPreviousVerb(token,verbs)
        updateFeats(token,'Tense','Past')
    elif xpos == 'CQ':
        headPartNextVerb(token,verbs)
        updateFeats(token,'PartType', 'Int')
    elif xpos == 'COND':
        headPartPreviousVerb(token,verbs)
        updateFeats(token,'PartType', 'Mod')
        updateFeats(token,'Mood', 'Cnd')
    elif xpos == 'PQ':
        headPartPreviousVerb(token,verbs)
        updateFeats(token,'PartType', 'Int')
    elif xpos == 'TOTAL':
        headPartPreviousVerb(token,verbs)
        # updateFeats(token,'PartType': 'Quant')
    elif xpos == 'CONS':
        headPartPreviousVerb(token,verbs)
        for feat,val in mapping['CONS'].items():
            updateFeats(token,feat, val)
    elif xpos == 'FOC':
        previous=PreviousContentWord(token,tokenlist)
        token['head']=previous['id']
        updateFeats(token,'PartType', 'Emp')
        updateFeats(token,'Foc', 'Yes')
    elif xpos == 'EXST' or xpos == 'PRSV':
        if len(verbs) > 0:
            headPartNextVerb(token,verbs)
        else:
            token['head']=0
            token['deprel']='root'
            verbs.append(token)
        value=mapping[xpos]['PartType']
        updateFeats(token,'PartType', value)

def handleVerb(token,nextToken,verbs):
    if nextToken['upos'] == 'NOUN': # TODO: possibly deprecated
        nextToken['deprel']='obj'
        nextToken['head']=token['id']
    elif nextToken['upos'] == 'SCONJ':
        token['deprel'] = 'advcl'
        headid = previousVerb(token,verbs)
        token['head'] = headid
    """else: # TODO:
        if len(verbs) > 1:
            if verbs[-1]['id'] == token['id']:
                token['head'] = verbs[0]['id']
                token['deprel'] = 'advcl'
                """

def previousPunct(token,tokenlist):
    puncts=TokensOfCatList(tokenlist,'PUNCT')
    return previousCat(token,puncts)

def getAdvHead(token,tokenlist,verbs):
    punctid=previousPunct(token,tokenlist)
    headid=previousVerb(token,verbs)
    tokenid=token['id']
    if not inInterval(headid,punctid,tokenid):
        headid=nextVerb(token,verbs)
    return headid

def AdvPronType(lastchar):
    for adv,prontype in ADVPRONTYPE.items():
        if adv.endswith(lastchar):
            return prontype
    return None

def WhAdvType(lastchar):
    for adv,advtype in WH_ADVTYPE.items():
        if adv.endswith(lastchar):
            return advtype
    return None

def RelativeInterrogativeAdv(tag):
    feats={}
    #feats['xpos']=tag
    prontype=None
    advtype=None
    if tag and tag.startswith('ADV') and len(tag) > 3:
        i=-1
        if len(tag) == 5:
            i=-2
            advtype=WhAdvType(tag[-1])
            if not advtype:
                advtype=ADVTYPE.get(tag[:-1])
                if tag.startswith('ADVD'):
                    lastchar=tag[-1]
                    feats['Deixis']=DEIXIS.get(lastchar)
        #xpos=f"{tag[0:3]}{tag[i]}"
        #feats['xpos']=xpos
        prontype=AdvPronType(tag[i])
    #if not prontype:
    #    prontype=
    feats['PronType']=prontype
    if not advtype:
        advtype=ADVTYPE.get(tag)
    feats['AdvType']=advtype
    return feats

def includeAdvType(token):
    xpos=token['xpos']
    dic=RelativeInterrogativeAdv(xpos)
    #key=dic['xpos']
    prontype=dic.get('PronType')
    advtype=dic.get('AdvType')
    deixis=dic.get('Deixis')
    if advtype:
        updateFeats(token,'AdvType',advtype)
    if prontype:
        updateFeats(token,'PronType',prontype)
    if deixis:
        updateFeats(token,'Deixis',deixis)

def handleAdv(token,nextToken, tokenlist,verbs):
    token['deprel']='advmod'
    previous=getPreviousToken(token,tokenlist,skip=['PUNCT'])
    xpos=token['xpos']
    #includeAdvType(token)
    feats=token.get('feats')
    prontype=None
    advtype=None
    if feats:
        prontype=feats.get('PronType')
        advtype=feats.get('AdvType')
    if prontype =='Rel':
        #updateFeats(token,'PronType', 'Rel')
        nouns=TokensOfCatList(tokenlist,'NOUN')
        nounid=previousCat(token,nouns)
        headid=nextVerb(token,verbs)
        token['head']=headid
        headlist=tokenlist.filter(id=headid)
        if headlist:
            head=headlist[0]
            head['deprel']='acl:relcl'
            if previous:
                head['head']=previous['id']
    elif prontype =='Deg':
        token['head']=previous['id']
        #updateFeats(token,'AdvType','Deg')
        feats=nextToken.get('feats')
        if feats and feats.get('PronType') == 'Rel':
            updateFeats(token,'Degree','Sup')
        else:
            updateFeats(token,'Degree','Cmp')
    elif prontype =='Int':
        token['head']=nextVerb(token,verbs)
        #updateFeats(token,'PronType', 'Int')
    else:
        token['head']=getAdvHead(token,tokenlist,verbs)
        #if xpos=='ADVD':
        #    updateFeats(token,'PronType', 'Dem')
        #    updateFeats(token,'AdvType', 'Loc')
    #includeAdvType(token)

def handleCconj(token,verbs):
    headid=nextVerb(token,verbs)
    token['deprel']='cc'
    token['head']=headid
    if headid:
        for verb in verbs:
            if verb['id'] == headid:
                verb['deprel']='conj'
                verb['head']=previousVerb(verb,verbs)
                break

def handleSconj(token,tokenlist,verbs):
    # token['deprel'] = 'mark'
    xpos=token['xpos']
    if xpos == 'SCONJR':
        token['head']=nextVerb(token,verbs)
    else:
        handleSconjS(token,tokenlist,verbs)
    headSconj(token,verbs) # TODO: redundant? See below.

def handleSconjS(token,tokenlist,verbs):
    '''TODO:
    1. adapt function to handle non-verbal clauses
    '''
    previous=getPreviousToken(token,tokenlist)
    if previous:
        if previous['xpos'] == 'NEG':
            token['head']=nextVerb(token,verbs)
        else:
            if previous['upos'] == 'PUNCT':
                token['head']=nextVerb(token,verbs)
            else:
                token['head']=previousVerb(token,verbs)
    else:
        token['head']=nextVerb(token,verbs)
    headSconj(token,verbs) # TODO: redundant? See above.

def headSconj(token,verbs):
    headid=token['head']
    headlist=verbs.filter(id=headid)
    if headlist:
        head=headlist[0]
        head['deprel']='advcl'
        head['head']=getHeadVerb(head,verbs)

def handlePosp(token,tokenlist,nouns):
    previous=getPreviousToken(token,tokenlist)
    if previous:
        if previous['upos'] == 'ADV':
            token['head']=previous['id']
            previous['deprel']='obl'
        elif previous['upos'] not in ('NOUN','PRON','PROPN'):
            nounid=previousCat(token,nouns)
            token['head']=nounid
            nounHead(nounid,nouns,tokenlist)

def nounHead(nounid,nouns,tokenlist):
    for noun in nouns:
        if noun['id'] == nounid:
            noun['deprel'] = 'obl'
            headlist=tokenlist.filter(deprel='root')
            head=previousCat(noun,headlist)
            noun['head'] = head
            break

def handlePrep(token,tokenlist,nouns):
    i=tokenlist.index(token)
    next=getNextToken(token,tokenlist[i+1:])
    nounid=0
    if next:
        upos=next['upos']
        if upos in ('NOUN','PRON','PROPN'):
            nounid=next['id']
            token['head']=nounid
        else:
            nounid=nextCat(token,nouns)
            token['head']=nounid
    nounHead(nounid,nouns,tokenlist)

def handleAdp(token,tokenlist,verbs):
    '''TODO:
    1. implement a function analogous to handleSconj and handleCconj
    2. change ADP treatment in the functions dealing with precedent categories
    3. does handleAdpCompl perform a similar job?
    '''
    feats=token.get('feats')
    if feats and feats.get('Rel') == 'NCont':
        token['deprel'] = 'obl'
        token['head'] = getHeadVerb(token,verbs)
    else:
        token['deprel'] = 'case'
        nouns=TokensOfCatList(tokenlist,'NOUN')
        if token['xpos'] == 'PREP':
            handlePrep(token,tokenlist,nouns)
        else:
            handlePosp(token,tokenlist,nouns)

def getHeadVerb(token,verbs):
    headid=previousVerb(token,verbs)
    if headid:
        return headid
    return nextVerb(token,verbs)

def previousVerb(token,verbs):
    tokenid=token['id']
    i=-1
    c=-len(verbs)
    while(i >= c):
        verbid=verbs[i]['id']
        if verbid < tokenid:
            return verbid
        i=i-1
    return 0

def nextVerb(token,verbs):
    tokenid=token['id']
    for verb in verbs:
        verbid=verb['id']
        if verbid > tokenid:
            return verbid
    return 0

def nextCat(token,cats):
    tokenid=token['id']
    for cat in cats:
        catid=cat['id']
        if catid > tokenid:
            return catid
    return 0

def getNextToken(token,tokenlist):
    for t in tokenlist:
        if t['form'] != token['form']:
            return t

def skipToken(upos,skip):
    return upos not in skip

def getPreviousToken(token,tokenlist,skip=[]):
    index=tokenlist.index(token)
    c=len(tokenlist[:index])
    if c > 1:
        i=c-1
        while(i >= 0):
            previous=tokenlist[i]
            if previous['form'] != token['form'] and skipToken(previous['upos'],skip):
                return previous
            i=i-1
    elif c == 1:
        return tokenlist[:index][0]

def previousCat(token,cats):
    tokenid=token['id']
    i=-1
    c=-len(cats)
    while(i >= c):
        catid=cats[i]['id']
        if catid < tokenid:
            return catid
        i=i-1
    return 0

def pronOrDet(token,nounid,verbs):
    verbid=nextVerb(token,verbs)
    tokenid=token['id']
    if verbid:
        if nounid > verbid or not nounid:
            token['upos']='PRON'
            setDeprel(token,verbid,'nsubj')
    elif not nounid:
        token['upos']='PRON'
        headid=previousVerb(token,verbs)
        setDeprel(token,headid,'obj')

def handlePronSeq(tokenlist):
    c=len(tokenlist)
    i=0
    while(i < c-1):
        token=tokenlist[i]
        if token['upos']=='PRON':
            next=tokenlist[i+1]
            if next['upos']=='PRON':
                if token['deprel'] == next['deprel']:
                    token['deprel']='det'
                    token['head']=next['id']
        i+=1

def handleDetNum(upos,token,nextToken,tokenlist,verbs):
    Pron='Pron'
    if upos == 'NUM':
        Pron='Num'
    xpos=token['xpos']
    value=getprontype(xpos)
    #value=xpos.title()
    deixis=DEIXIS.get(xpos)
    if deixis:
        updateFeats(token,'Deixis',deixis)
        #value=value[:3]
    updateFeats(token,f'{Pron}Type',value)
    if token['feats'].get('PronType') == 'Art':
        updateFeats(token,'Definite','Ind')
    feats=nextToken.get('feats')
    if feats:
        rel=feats.get('Rel')
        if rel == 'Cont':
            token['upos']='PRON'
            token['head']=nextToken['id']
            token['deprel']='nmod:poss'
    deprel={'DET': 'det', 'NUM': 'nummod'}
    delta=2
    nouns=TokensOfCatList(tokenlist,'NOUN')
    nounid=nextCat(token,nouns)
    pronOrDet(token,nounid,verbs)
    deprel=deprel.get(token['upos'])
    if deprel:
        token['deprel']=deprel
        tokenid=token['id']
        if nounid - tokenid > delta:
            token['head']=getNextWord(token,tokenlist)['id']
        else:
            token['head']=nounid
    else:
        if nextToken['upos'] == 'ADP':
            handleAdpCompl(token,verbs,nextToken)

def getNextWord(token, tokenlist):
    start=tokenlist.index(token)
    i=start + 1
    while (i < len(tokenlist)):
        t=tokenlist[i]
        if t['upos'] != 'PUNCT' and t['form'] != token['form']:
            return t
        i+=1

def headAux(verb,headid):
    if headid:
        verb['upos'] = 'AUX'
        verb['deprel'] = 'aux'
        verb['head'] = headid

def setUposXpos(verb,pos):
    verb['xpos'] = pos
    verb['upos'] = UDTAGS[pos]

def inInterval(num,begin,end):
    return begin < num < end

def sameClause(headid,previous,next):
    return inInterval(headid,previous,next)

def handleSameClause(verb,pos,headid,previous,next):
    if sameClause(headid,previous,next):
        headAux(verb,headid)
        setUposXpos(verb,pos)

def isCop(token):
    return token['lemma'] == 'ikú'

def mkCop(token):
    #token['deprel'] = 'cop' 070723
    token['xpos'] = 'COP'
    #token['upos'] = 'AUX' 070723

def AdjAdvorOblRoot(root,cop,overwrite=True):
    rootid=root['id']
    setDeprel(cop,rootid,'cop',overwrite)
    cop['upos']='AUX'
    root['deprel']='root'
    root['head']=0

def getAdjAdvRoot(tokenlist,cop,deps,overwrite):
    rootid=None
    for upos in ['ADJ','ADV']:
        rootid=getUposRoot(tokenlist,upos,cop,deps,overwrite)
        if rootid != None:
            return rootid

def getUposRoot(tokenlist,upos,cop,deps,overwrite):
    for token in tokenlist:
        if token['upos'] == upos:
            AdjAdvorOblRoot(token,cop,overwrite)
            rootid=token['id']
            if token in deps:
                deps.remove(token)
            return rootid

def handleCopClause(tokenlist,copulas):
    cop=copulas[0]
    copid=cop['id']
    rootid=copid
    deps=tokenlist.filter(head=copid)
    obls=deps.filter(deprel='obl')
    for obl in obls:
        if obl['id'] > copid:
            AdjAdvorOblRoot(obl,cop)
            rootid=obl['id']
            deps.remove(obl)
            break
    if rootid == copid:
        advmods=deps.filter(deprel='advmod')
        for advmod in advmods:
            if advmod['id'] > copid:
                AdjAdvorOblRoot(advmod,cop)
                rootid=advmod['id']
                deps.remove(advmod)
                break
    if rootid == copid:
        tokenid=getAdjAdvRoot(tokenlist,cop,deps,True)
        if tokenid:
            rootid=tokenid
    for dep in deps:
        dep['head']=rootid

def isAdvCl(verb,tokenlist,delta=1):
    verbid=verb['id']
    sconjs=verbid+delta
    return tokenlist.filter(id=sconjs)

def handleAux(tokenlist):
    verbs=VerbIdsList(tokenlist)
    puncts=TokensOfCatList(tokenlist,'PUNCT')
    c=len(verbs)
    if c == 1:
        if isCop(verbs[0]):
            mkCop(verbs[0])
            # handleNonVerbalRoot()
    elif c > 1:
        for verb in verbs:
            verbid=verb['id']
            lemma=verb['lemma']
            entry=extractAuxEntry(lemma)
            pos=entry.get('pos')
            previous=previousCat(verb,puncts)
            next=nextCat(verb,puncts)
            headid=0
            if pos == 'AUXFR':
                i=verbs.index(verb)
                if i+1 < len(verbs):
                    nextverb=verbs[i+1]
                    feats=nextverb.get('feats')
                    if feats and feats.get('Compound'):
                        pass
                    else:
                        headid=nextVerb(verb,verbs)
                        handleSconjList(tokenlist,verbs)
                        advcl=tokenlist.filter(head=headid,deprel='mark')
                        #if not isAdvCl(verb,tokenlist):
                        if not advcl:
                            handleSameClause(verb,pos,headid,previous,next)
            elif pos == 'AUXFS' or pos == 'AUXN':
                headid=previousVerb(verb,verbs)
                handleSameClause(verb,pos,headid,previous,next)
            else:
                pass # TODO: ccomp, xcomp, advcl
    for verb in verbs:
        if verb['upos'] == 'AUX':
            if verb['lemma'] == 'yuíri':
                updateFeats(verb,'Aspect','Iter')

def handleNCont(upos,feats):
    if feats['Rel'] == 'NCont':
        if upos == 'VERB':
            feats.update({'Number':'Sing',
            'Person' : '3',
            'VerbForm' : 'Fin'})
        elif upos == 'NOUN':
            feats.update({'Number[psor]':'Sing',
            'Person[psor]' : '3'})
        elif upos == 'ADP':
            feats.update({'Number[grnd]':'Sing',
            'Person[grnd]' : '3'})

def extractClauses(tokenlist):
    clauses=[]
    for token in tokenlist:
        if token['form'] == ',':
            first=TokenList()
            index=tokenlist.index(token)
            first.extend(tokenlist[:index])
            clauses.append(first)
            last=TokenList()
            last.extend(tokenlist[index:])
            clauses.append(last)
            break
    return clauses

def assignHead(tokenlist,rootid):
    for token in tokenlist:
        if token['head'] == 0 and token['deprel'] != 'root':
            token['head'] = rootid

def isNounModifier(token):
    return token['uops'] in ('DET','PRON')

def assignSubj(rootid,nouns):
    i=-1
    while(i>=-len(nouns)):
        noun=nouns[i]
        if noun['id'] < rootid:
            setAttribute(noun,'deprel','nsubj')
            setAttribute(noun,'head',rootid)
            break
        i=i-1

def handleOblRoot(tokenlist):
    copulas=tokenlist.filter(xpos='COP')
    if copulas:
        handleCopClause(tokenlist,copulas)
    else:
        verbs=tokenlist.filter(upos='VERB')
        if not verbs:
            handleOblRootNoCop(tokenlist)

def handleOblRootNoCop(tokenlist):
    obls=tokenlist.filter(deprel='obl')
    puncts=tokenlist.filter(deprel='punct')
    if obls:
        nouns=extractNominals(tokenlist)
        root=obls[-1]
        root['deprel']='root'
        root['head']=0
        rootid=root['id']
        if puncts:
            puncts[-1]['head']=rootid
        assignSubj(rootid,nouns)

def extractNominals(tokenlist):
    nouns=TokensOfCatList(tokenlist,'NOUN')
    nouns.extend(TokensOfCatList(tokenlist,'PRON'))
    nouns.extend(TokensOfCatList(tokenlist,'PROPN'))
    return nouns

def AdjOrNounRoot(tokenlist):
    adjs=TokensOfCatList(tokenlist,'ADJ')
    nouns=extractNominals(tokenlist)
    #cop=tokenlist.filter(deprel='cop') 070723
    cop=tokenlist.filter(xpos='COP')
    rootid=0
    if adjs:
        root=adjs[-1]
        setAttribute(root,'deprel','root')
        rootid=root['id']
        if nouns:
            i=-1
            while(i>=-len(nouns)):
                noun=nouns[i]
                if noun['id'] < rootid:
                    setAttribute(noun,'deprel','nsubj')
                    setAttribute(noun,'head',rootid)
                    break
                i=i-1
    if cop:
        cop[0]['head'] = rootid
    assignHead(tokenlist,rootid)

def GenitiveConstruction(nouns,tokenlist):
    i=0
    c=len(nouns)-1
    while(i<c):
        this=nouns[i]
        if not this['deprel']:
            for token in tokenlist[this['id']:]:
                if token['upos'] in ('AUX','VERB','ADV'):
                    return
            j=i+1
            if j <=c:
                this['deprel']='nmod:poss'
                next=nouns[j]
                this['head']=next['id']
        i+=1

def handleExpletive0(tokenlist): # TODO: handle non-adjacent subject
    i=1
    while(i<len(tokenlist)):
        this=tokenlist[i]
        previous=tokenlist[i-1]
        if this['xpos'] == 'PRON2' and this['deprel'] == 'nsubj':
            if previous['deprel'] == 'nsubj':
                this['deprel'] = 'expl'
        i+=1

def handleExpletive(tokenlist): # TODO: handle non-adjacent subject
    i=1
    while(i<len(tokenlist)):
        this=tokenlist[i]
        if this['xpos'] == 'PRON2' and this['deprel'] == 'nsubj':
            headid=this['head']
            for token in tokenlist[:i]:
                if token['head'] == headid and token['deprel'] == 'nsubj':
                    this['deprel'] = 'expl'
                    break
        i+=1

def handlePunct(token,nextToken,tokenlist,verbs):
    if nextToken['lemma'] in DEPPUNCT:
        nextToken['head'] = nextVerb(token,verbs)
    elif nextToken['xpos'] == 'ELIP':
         updateFeats(nextToken,'PunctType','Elip')
         if token['upos'] == 'PUNCT':
             rootlist=tokenlist.filter(deprel='root')
             if rootlist:
                 token['head']=rootlist[0]['id']
                 nextToken['head'] = token['head']
         else:
            nextToken['head'] = previousVerb(nextToken,verbs)
    elif nextToken['lemma'] not in QUOTES:
        headlist=tokenlist.filter(deprel='root')
        head=1
        if headlist:
            head=headlist[0]['id']
        nextToken['head']=head

def handleComma(token,tokenlist):
    verbs=VerbsList(tokenlist)
    token['head'] = nextVerb(token,verbs)

def handleStartPunct(tokenlist):
    first=tokenlist[0]
    if first['upos'] == 'PUNCT' and first['lemma'] in DASHES:
        root=tokenlist.filter(deprel='root')
        if root:
            first['head']=root[0]['id']

def mkPunctToken(punct,start,ident):
    parselist=getparselist(punct)
    entries=extract_feats(parselist)
    return mkConlluToken(punct,entries[0],start=start,ident=ident)

def insertPunct(punct,tokenid,tokenlist):
    position=tokenid-1
    previous=tokenlist.filter(id=position)[0]
    insertNoSpaceAfter(previous)
    start=int(getStartEnd(previous,False)['end'])
    token=mkPunctToken(punct,start,tokenid)
    incrementSentId(tokenlist,tokenid)
    tokenlist.insert(position,token)
    comma=tokenlist.filter(id=tokenid)[0]
    handleComma(comma,tokenlist) # TODO: adapt this to hanlde other punctuation
    incrementTokenRange(tokenlist[tokenid+1:])

def incrementSentId(tokenlist,startid):
    for token in tokenlist:
        if token['id'] >= startid:
            token['id']+=1
        if token['head'] >= startid:
            token['head']+=1

def handleRel(token,tokenlist):
    token['deprel'] = 'nsubj'
    i=len(tokenlist)
    previous=getPreviousToken(token,tokenlist)
    if previous['upos'] == 'PART':
        previous=getPreviousToken(previous,tokenlist)
    token['head'] = previous['id']
    nouns=TokensOfCatList(tokenlist,'NOUN')
    prons=TokensOfCatList(tokenlist,'PRON')
    for pron in prons:
        if pron['xpos'] == 'PRON2':
            prons.remove(pron)
    tokenid=token['id']
    headids=[tokenid]
    nounid=previousCat(token,nouns)
    headids.append(nounid)
    pronid=previousCat(token,prons)
    headids.append(pronid)
    headids.sort()
    j=headids.index(tokenid)
    headid=nounid
    if headids.index(tokenid) > 0:
        headid=headids[j-1]
    previous['head'] = headid
    previous['deprel'] = 'acl:relcl'

def handleVerbs(verbs):
    rootlist=verbs.filter(deprel='root')
    if len(rootlist) > 1:
        for verb in rootlist[1:]:
            headid = previousVerb(verb,verbs)
            setDeprel(verb, headid,deprel='parataxis',overwrite=True)

def handleSconjList(tokenlist,verbs):
    for token in tokenlist:
        if token['upos'] == 'SCONJ':
            handleSconj(token,tokenlist,verbs)

def addFeatures(tokenlist):
    start=0
    i=0
    c=len(tokenlist) -1
    handleAux(tokenlist)
    verbid=FirstVerbId(tokenlist)
    verbs=VerbIdsList(tokenlist)
    if not verbs:
        start=handleNmodPoss(tokenlist)
        AdjOrNounRoot(tokenlist)
    while(i < c) :
        token=tokenlist[i]
        upos=token['upos']
        nextToken=getNextToken(token, tokenlist[i+1:])
        if upos in ('NOUN','PRON','PROPN'):
            if token['xpos'] == 'REL':
                handleRel(token,tokenlist)
            else:
                handleNounPron(token,nextToken,verbs)
                if upos == 'PRON':
                    handlePron(token,nextToken)
        elif upos == "PART":
            handlePart(token,tokenlist,verbs)
        elif upos == "INTJ":
            handleIntj(token,verbs)
        elif upos == "VERB":
            pass
            # TODO: handleVerb(token,nextToken,verbs)
        elif upos == "ADP":
            handleAdp(token,tokenlist,verbs)
        elif upos == "SCONJ": # TODO: iterate over tokenlist annotating all SCONJs
            handleSconj(token,tokenlist,verbs)
        elif upos == "CCONJ":
            handleCconj(token,verbs)
        elif upos == "ADV":
            handleAdv(token,nextToken,tokenlist,verbs)
        elif upos in ("DET","NUM"):
            handleDetNum(upos,token,nextToken,tokenlist,verbs)
        if nextToken and nextToken['upos'] == 'PUNCT': # TODO: sentences without final punctuation
            handlePunct(token,nextToken, tokenlist,verbs)
        i+=1
    handleVerbs(verbs)
    handleNmodPoss(tokenlist[start:])

def mkVocative(token,punct):
    if punct['lemma'] == COMMA:
        token['deprel'] = 'vocative'
        punct['head'] = token['id']

def handleVocative(tokenlist):
    for token in tokenlist:
        if token['deprel'] in ('nsubj','obj'):
            i=tokenlist.index(token)
            next=getNextToken(token,tokenlist[i+1:])
            previous=getPreviousToken(token,tokenlist)
            if next and next['upos'] == 'PUNCT':
                if previous:
                    if previous['upos'] == 'PUNCT':
                        mkVocative(token,previous)
                        break
                    elif previous['deprel'] == 'nmod:poss':
                        beforeprevious=getPreviousToken(previous,tokenlist)
                        if beforeprevious:
                            if beforeprevious['upos'] == 'PUNCT':
                                mkVocative(token,beforeprevious)
                                break
                        else:
                            mkVocative(token,next)
                            break
                else:
                    mkVocative(token,next)
                    break

def getXpos(parse):
	tags=parse[1]
	if tags:
		return tags.split('+')[0]
	return ''

def getTags(parse):
    tags=''
    m=None
    if parse[1]:
        m=TAGSEQ.search(parse[1])
    if m:
        x,y=m.groups()
        if x:
            tags=f"{x}{y}"
        else:
            tags=y
    return tags

def hasTag(tags1,tags2):
    if re.search(rf"\b{re.escape(tags2)}\b",tags1):
        return True
    return False

def filterparselist(tags,parselist):
    return list(filter(lambda x: hasTag(getTags(x),tags.upper()),parselist))

def handleCompoundAux(token):
    updateFeats(token,'Compound','Yes')

def handleClitic(token):
    updateFeats(token,'Clitic','Yes')

def handleHyphenSepToken(token):
    upos=token['upos']
    if upos == 'VERB':
        handleCompoundAux(token)
    else:
        handleClitic(token)

def getStartEnd(token,remove=True):
    dic={}
    misc=token.get('misc')
    if misc:
        tokenrange=''
        if misc.get('TokenRange'):
            if remove:
                tokenrange=misc.pop('TokenRange')
            else:
                tokenrange=misc.get('TokenRange')
        if not misc:
            token['misc']=None
        if tokenrange:
            start,end=tokenrange.split(':')
            dic['start']=start
            dic['end']=end
    return dic

def getSpaceAfter(token):
    misc=token.get('misc')
    if misc.get('SpaceAfter'):
        return misc.pop('SpaceAfter')

def insertMultitokenWord(tokenlist):
    newlist=TokenList()
    sep='-'
    for token in tokenlist:
        feats=token.get('feats')
        if feats:
            if feats.get('Compound') == 'Yes'or feats.get('Clitic') == 'Yes':
                index=tokenlist.index(token)
                previous=tokenlist[index-1]
                startend=getStartEnd(previous)
                start=startend.get('start')
                if start:
                    spaceafter=getSpaceAfter(token)
                    end=getStartEnd(token)['end']
                    alomorph=''
                    misc=previous['misc']
                    if misc:
                        alomorph=misc.get('Alomorph')
                    first=previous['form']
                    if alomorph:
                        first=alomorph
                    #if token['upos'] == 'ADV' or token['form'] == ME:
                    #    sep=''
                    mwt=MULTIWORDTOKENS.get(first)
                    if mwt:
                        form=mwt
                    else:
                        form=f"{first}{sep}{token['form']}"
                    tokenid=token['id']
                    ident=f'{tokenid-1}-{tokenid}'
                    compound=mkMultiWordToken(ident,form,start,end,spaceafter)
                    tokenlist.insert(index-1,compound)
                #break

def handleHyphen(form):
    dic={}
    dic['form']=form
    dic['hyphen']=False
    if form.startswith(HYPHEN):
        dic['form']=form[1:]
        if form[1:] in CLITICS:
            dic['lemma']='upé'
            dic['xpos']='ADP'
        dic['hyphen']=True
    elif form.endswith(HYPHEN):
        dic['host']=True
        dic['form']=form[:-1]
    elif form.startswith(UNDERSCORE):
        mkSuff(form,dic)
    return dic

def mkNTU(dic): # TODO: remove if deprecated
    dic['form']=form[1:]
    dic['lemma']='ntu'
    dic['upos']='ADV'
    dic['underscore']=True

def mkSuff(form,dic):
    form=form[1:]
    ntu={'xpos':'ADV','lemma':'ntu','clitic': NTU}
    me={'xpos':'ADP','lemma':'upé','clitic':ME}
    ta={'xpos': 'CQ','lemma':'taá','clitic':TA}
    suffs=[ntu,me,ta]
    for suff in suffs:
        clitic=suff.get('clitic')
        if clitic == form:
            dic['form']=form
            dic['upos']=getudtag(suff['xpos'])
            dic['xpos']=suff['xpos']
            dic['lemma']=suff['lemma']
            dic['underscore']=True
            break

def mkPropn(token,orig=None,orig_form=None):
    new={}
    feats=['PROPN']
    tags='+'.join(feats)
    lemma=token.lower()
    if orig:
        new['OrigLang']=orig
    if orig_form:
        new['Orig']=orig_form
    new['parselist']=[[lemma, tags]]
    return new

def mkAdv(form):
    return [[form.lower(), 'ADV']]

def mkVerb(form,derivation='',orig=None, orig_form=None):
    new={}
    feats=['V']
    if derivation:
        feats.append(derivation)
    if orig:
        new['OrigLang']=orig
    if orig_form:
        new['Orig']=orig_form
    entry=guessVerb(form)
    feats.append(entry['person'])
    number=entry.get('number')
    if number:
        feats.append(number)
    tags='+'.join(feats)
    lemma=entry['lemma']
    new['parselist']=[[lemma, tags]]
    return new

def handleAccent(base,nasal=False,force=False):
    parselist=getparselist(base)
    tags=list(filter(lambda x: x[1],parselist))
    if tags and not force:
        return base
    return accent(base,nasal)

def endswith(form,suff):
    if form.endswith(suff):
        return form[:-len(suff)]
    return ''

def mkHab(form): # TODO: this seems deprecated (see mkHabXpos)
    suff='tiwa'
    if form.endswith(suff):
        base=form[:-len(suff)]
        form=handleAccent(base)
        return mkVerb(form,derivation='HAB')

def mkHabSconj(form): # TODO: this seems deprecated (see mkHabXpos)
    suff='tiwa'
    new={}
    if form.endswith(suff):
        base=form[:-len(suff)]
        lemma=handleAccent(base)
        new['parselist']=[[lemma, 'SCONJ+HAB']]
        return new

def parseWord(form,lenght):
    base=form[:-lenght].lower()
    suff=form[-lenght:]
    return base,suff

def getval(key,dic):
    for k,v in dic.items():
        if k.startswith(key):
            return v
    return None

def mkHabXpos(form,xpos='', lenght=0, accent=False, guess=False, force=False):
    if not lenght:
        lenght=4
    suffs={'wara':{'Aspect':'FREQ', 'Tense': 'PRES'},
    'tiwa': {'Aspect':'HAB'},
    'wera': {'Aspect':'FREQ', 'Tense': 'PAST'}}
    base, suff=parseWord(form,lenght)
    tag=suffs[suff]['Aspect']
    feats=[]
    feats.append(tag)
    tense=suffs[suff].get('Tense')
    if tense:
        feats.append(tense)
    new={}
    if accent:
        base=handleAccent(base,force=force)
    if not xpos:
        xpos='V'
    if not guess:
        parselist=getparselist(base)
        parses=filterparselist(xpos,parselist)
        if parses:
            parse=parses[0]
            parse[1]=f"{parse[1]}+{'+'.join(feats)}"
            new['parselist']=[parse]
            return new
    if xpos == 'V':
        new=mkVerb(base,derivation=tag)
    elif xpos == 'N':
        pass
    else:
        new['parselist']=[[base, f"{xpos}+{tag}"]]
    return new

def mkNoun(form,orig=None,dic={},orig_form=''):
    feats=[]
    new={}
    lemma=form.lower()
    if orig:
        new['OrigLang']=orig
    if orig_form:
        new['Orig']=orig_form
    number=dic.get('number')
    degree=dic.get('degree')
    derivation=dic.get('derivation')
    if degree:
        feats.append(degree)
    if derivation:
        feats.append(derivation)
    if not number:
        d=getNumber(lemma)
        number=d['number']
        lemma=d.get('lemma')
    feats.append(number)
    new['parselist']=[[lemma, f"N+{'+'.join(feats)}"]]
    return new

def mkAdj(form,orig='pt',dic={},orig_form=''):
    form=form.lower()
    feats=[]
    new={}
    if orig:
        new['OrigLang']=orig
    if orig_form:
        new['Orig']=orig_form
    degree=dic.get('degree')
    derivation=dic.get('derivation')
    if degree:
        feats.append(degree)
    if derivation:
        feats.append(derivation)
    new['parselist']=[[form, f"A+{'+'.join(feats)}"]]
    return new

def getNumber(form):
    dic={}
    dic['number']='SG'
    if form.endswith('-itá'):
        dic['number']='PL'
        dic['lemma']=form[:-4]
    else:
        dic['lemma']=form
    return dic

def mkAug(form,force=False): # TODO: superseded by mkEval
    form=form.lower()
    i=-4
    dic=getNumber(form)
    dic['degree']='AUG'
    number=dic.get('number')
    if number == 'PL':
        i=-8
    lemma=handleAccent(form[:i],force=force)
    return mkNoun(lemma,None,dic)

def mkEval(form,xpos='N',force=False):
    suffixes={'wasú': 'AUG', 'mirĩ': 'DIM', 'í': 'DIM'}
    xpos=xpos
    dic={}
    dic['lemma']=form.lower()
    if xpos=='N':
        dic.update(getNumber(dic['lemma']))
    for suff,feat in suffixes.items():
         if dic['lemma'].endswith(suff):
             dic['degree']=feat
             dic['lemma']=dic['lemma'][:-len(suff)]
             break
    lemma=handleAccent(dic['lemma'],force=force)
    if dic.get('number'):
        return mkNoun(lemma,None,dic)
    return mkAdj(lemma,None,dic)

def mkCol(form):
    form=form.lower()
    suff='tiwa'
    i=-len(suff)
    dic=getNumber(form)
    dic['derivation']='COL'
    number=dic.get('number')
    if number == 'PL':
        i-=-4
    lemma=handleAccent(form[:i])
    return mkNoun(lemma,None,dic)

def mkPrv(form, xpos='A'):
    form=form.lower()
    i=-3
    dic={}
    tag='PRV'
    dic['derivation']=tag
    lemma=handleAccent(form[:i])
    new={}
    if xpos == 'A':
        new=mkAdj(lemma,None,dic)
    elif xpos == 'A2':
        new['parselist']=[[lemma, f"{xpos}+{tag}"]]
    return new

def mkClitic(lemma,upos):
    return [[lemma, upos]]

def mkUpos(lemma,upos):
    return [[lemma.lower(), upos]]

def mkIntj(lemma):
    return mkUpos(lemma,'INTJ')

def handleRedup(token):
    redup={}
    if HYPHEN in token and not token in LEXREDUP:
        parts=token.split('-')
        if len(parts) == 2 and parts[1]:
            if parts[0].endswith(parts[1]):
                redup['form'],redup['lemma']=parts[0],parts[1]
    return redup

def handleAlomorph(form):
    return form.replace('ií','iwa')

def mkRoot(index):
    ROOT.append(index)

def isMultitokenWord(tokenid):
	return (type(tokenid) is tuple and len(tokenid) == 3) or (type(tokenid) is str and len(tokenid.split('-')) == 2)

def setRoot(tokenlist):
    rootlist=tokenlist.filter(deprel='root')
    if rootlist:
        root=rootlist[0]
        rootid=root['id']
        for token in tokenlist:
            tokenid=token['id']
            if not isMultitokenWord(tokenid) and tokenid != rootid:
                if not token['head']:
                    token['head'] = rootid

def handleRoot(tokenlist):
    rootid=0
    if ROOT:
        rootid=ROOT.pop()+1
        token=tokenlist.filter(id=rootid)[0]
        token['deprel']='root'
        token['head']=0
        feats=token.get('feats')
        isIntPron=False
        if feats:
            if feats.get('PronType') == 'Int':
                isIntPron=True
        for t in tokenlist:
            if t['id'] != rootid:
                if t['head'] == 0:
                    t['head']=rootid
                    if isCop(t):
                        mkCop(t)
                if not t['deprel']:
                    if isNominal(t['upos']):
                        if t['id'] < rootid or isIntPron:
                            setAttribute(t,'deprel','nsubj')
        last=tokenlist[-1]
        if last['deprel'] == 'punct' and last['head'] != rootid:
            last['head'] = rootid

def isNominal(upos):
    return upos in ['NOUN','PROPN','PRON']

def parseArgs(string):
    dic={}
    #oper='='
    #index=string.index(oper)
    #parts=string[index+1:].split(':')
    parts=string.split(':')
    dic['func'] = parts[0]
    for part in parts[1:]:
        arg,val=part.split('|')
        dic[arg]=val
    return dic

def parseTag(tag):
    values={'t': True, 'f': 'False'}
    sep=':'
    dic={}
    if sep in tag:
        args=parseArgs(tag)
        for k,v in args.items():
            value = values.get(v)
            if value:
                args[k]=values[v]
            else:
                if k == 'l':
                    args[k]=int(v)
        dic.update(args)
    else:
        dic['func']=tag
    return dic

def insertRedup(parselist):
    for parse in parselist:
        lemma,tags=parse[0],parse[1]
        if tags:
            taglist=tags.split('+')
            c=len(taglist)
            if c > 1:
                taglist.insert(1,REDUP)
            elif c == 1:
                taglist.append(REDUP)
            parse[1]="+".join(taglist)

def extractTag(token):
    if '/' in token:
        return token.split('/')

def mkConlluSentence(tokens):
    ROOT.clear()
    tokenlist=TokenList()
    ident=1
    start=0
    for token in tokens:
        old=token
        tag=MULTIWORDTOKENS.get('xpos')
        root=False
        parts=extractTag(token)
        if parts:
            token,tag=parts
            if '@' in tag:
                root=True
                tag=tag[:-1]
        dic=handleHyphen(token)
        new={}
        form=dic.get('form')
        redup=handleRedup(token)
        parselist=[]
        lemma=dic.get('lemma')
        upos=dic.get('upos') # TODO: problably not used
        xpos=dic.get('xpos')
        host=dic.get('host')
        alomorph=form.lower()
        if redup:
            parselist=getparselist(redup['form'])
            insertRedup(parselist)
        else:
            if lemma:
                parselist=mkClitic(lemma,xpos)
            else:
                if host:
                    alomorph=handleAlomorph(form.lower())
                    parselist=getparselist(alomorph)
                else:
                    parselist=getparselist(form.lower())
        if tag:
            tagparse=parseTag(tag)
            t=tagparse.get('func')
            if t:
                tag=t
            orig=tagparse.get('o')
            orig_form=tagparse.get('s')
            force=tagparse.get('f')
            xpos=tagparse.get('x')
            if xpos:
                xpos=xpos.upper()
            newparselist=[]
            if tag == '=p':
                new=mkPropn(token,orig=orig,orig_form=orig_form)
                newparselist=new['parselist']
            elif tag == '=adv':
                newparselist=mkAdv(token)
            elif tag == '=n':
                new=mkNoun(form,orig=orig,orig_form=orig_form)
                newparselist=new['parselist']
            elif tag == '=a':
                new=mkAdj(form,orig=orig,orig_form=orig_form)
                newparselist=new['parselist']
            elif tag == '=v':
                new=mkVerb(form,orig=orig,orig_form=orig_form)
                newparselist=new['parselist']
            elif tag == '=hab': # TODO hab -> ohab
                new=mkHab(form)
                newparselist=new['parselist']
            elif tag == '=col':
                new=mkCol(form)
                newparselist=new['parselist']
            elif tag == '=hab=sconj':
                new=mkHabSconj(form)
                newparselist=new['parselist']
            elif tag == '=mkhab': # TODO mkhab -> hab
                new=mkHabXpos(  form,
                                xpos,
                                lenght=tagparse.get('l'),
                                accent=tagparse.get('a'),
                                guess=tagparse.get('g'),
                                force=force
                                )
                newparselist=new['parselist']
            elif tag == '=aug':
                new=mkAug(form,force)
                newparselist=new['parselist']
            elif tag == '=ev':
                new=mkEval(form,xpos,force)
                newparselist=new['parselist']
            elif tag == '=prv':
                new=mkPrv(form,xpos)
                newparselist=new['parselist']
            elif tag == '=intj':
                newparselist=mkIntj(form)
            #elif tag == '=r':
            #    mkRoot(tokens.index(old))
            else:
                newparselist=filterparselist(tag,parselist)
            if newparselist:
                parselist=newparselist
        if root:
            mkRoot(tokens.index(old))
        entries=extract_feats(parselist)
        for entry in entries:
            if entry.get('pos') == 'PUNCT':
                start=start-1
            t=mkConlluToken(form,entry,start=start, ident=ident)
            if dic.get('hyphen') or dic.get('underscore'):
                handleHyphenSepToken(t)
            if new:
                orig=new.get('OrigLang')
                orig_form=new.get('Orig')
                if orig:
                    t['misc'].update({'OrigLang': orig})
                if orig_form:
                    t['misc'].update({'Orig': orig_form})
            if host and form.lower() != alomorph:
                t['misc'].update({'Alomorph': form})
            tokenlist.append(t)
        start=start+len(token)+1
        ident+=1
    handleSpaceAfter(tokenlist)
    addFeatures(tokenlist)
    insertMultitokenWord(tokenlist)
    handleOblRoot(tokenlist)
    handleRoot(tokenlist)
    handleExpletive(tokenlist)
    handleVocative(tokenlist)
    handleStartPunct(tokenlist)
    setRoot(tokenlist)
    handlePronSeq(tokenlist)
    sortTokens(tokenlist)
    return tokenlist

def extractConlluSents(infile):
    sentences=[]
    data_file = open(infile, "r", encoding="utf-8")
    for tokenlist in parse_incr(data_file):
        sentences.append(tokenlist)
    return sentences

def insertSentId(sent,pref='MooreFP1994',textid=0,sentid=1):
    sent.metadata.update({'sent_id': f'{pref}:{textid}:{sentid}'})
    sent.metadata=sortDict(sent.metadata)

def insertAnnotator(sentences,annotator):
    for sent in sentences:
        sent.metadata.update({'text_annotator':annotator})

def extract_sents(line=None,lines=None):
    sents=[]
    if lines:
        for sent in lines.split("\n"):
            sents.append(sent.strip())
    else:
        sents=[sent.strip() for sent in PARTS.split(line,4) if sent]
    return sents

def ppText(sents,pref='',textid=0,index=0,sentid=0):
    output=[]
    if pref:
        output.append(f"# sent_id = {pref}:{textid}:{index}:{sentid}")
    yrl,eng,por=sents[0],sents[1],sents[2]
    template=f"# text = {yrl}\n# text_eng = {eng}\n# text_por = {por}"
    dic={}
    if len(sents) > 3:
        for sent in sents[3:]:
            if sent.startswith('(') and sent.endswith(')'):
                dic['source']=sent[1:-1]
            else:
                dic['orig']=sent
    output.append(template)
    for k,v in dic.items():
        output.append(f"# text_{k} = {v}")
    return '\n'.join(output)

def mkText(text):
	print(ppText(extract_sents(lines=text)))

def extractYrl(sent):
    return REMOVE.sub('',sent)

def handleSents(sents,pref,textid,index,sentid,annotator):
    yrl=extractYrl(sents[0])
    sents[1]=f"({sents[1]})"
    output=[]
    if len(sents) == 5:
        output.append(ppText([yrl,sents[3],
        sents[2],
        sents[1],
        sents[4]],
        pref,textid,index,sentid))
    else:
        output.append(ppText([yrl,
        sents[3],
        sents[2],
        sents[1]],
        pref,textid,index,sentid))
    tk=parseSentence(sents[0])
    includeAnnotator(output,annotator)
    output.append(tk.serialize())
    return output

def includeAnnotator(output,annotator):
    output.append(f"# text_annotator = {annotator}")

def formatList(output):
    return '\n'.join(output)

def handleParse(outstring,copyboard=True):
    if copyboard:
        import pyperclip
        pyperclip.copy(outstring)
    print(outstring)

def getFileNameParts(pref,textid,index,sentid):
    return {'pref':pref, 'textid':str(textid),
    'index':str(index),'sentid':str(sentid)}

def TreebankSentence(text='',pref='',textid=0,index=0,sentid=0,copyboard=True,annotator='LFdeA',outfile=False,overwrite=False):
    if not text:
        text='''Aité kwá sera waá piranha yakunheseri
        aé i turususá i apuã waá rupí, asuí sanha
        saimbé yuíri. (Payema, 68, adap.)
        - Este que se chama piranha nós o conhecemos
        por seu formato oval e por seus dentes afiados.
        - This one called piranha we know for its oval shape
        and its sharp teeth.'''.replace('\n','') # Avila (2021, p. 256)
        text=re.sub(r"\s+",' ',text)
    sents=extract_sents(text)
    output=handleSents(sents, pref,textid,index,sentid,annotator)
    outstring=formatList(output)
    if outfile:
        metadata=getFileNameParts(pref,textid,index,sentid)
        saveParseToFile(includeText(text,outstring),metadata,overwrite)
    handleParse(outstring,copyboard=copyboard)

def saveParseToFile(outstring,metadata,overwrite):
    values=list(metadata.values())
    filename=f"{'-'.join(values)}.conllu"
    if os.path.exists(filename) and not overwrite:
        print(f"{filename} already exists.")
    else:
        outfile=open(filename,'w')
        print(outstring,file=outfile)
        outfile.close()

def includeTranslation(example):
    from deep_translator import GoogleTranslator
    parts=extract_sents(example)
    i=-1
    if len(parts) == 4:
        i=-2
    text_por=parts[i]
    text_eng=GoogleTranslator(source='pt', target='en').translate(text_por)
    if len(parts) == 4:
        parts.insert(-1,text_eng)
    else:
        parts.append(text_eng)
    return parts

def parseExample(example,pref,textid,index,sentid,copyboard=True,annotator=ANNOTATOR,check=True, outfile=False, overwrite=False):
    yrl=extract_sents(example)[0] # TODO: avoid calling this function three times
    if check:
        if checkSentence(yrl):
            print(f"Sentence '{yrl}' already is in the treebank.")
            return
    sents=includeTranslation(example)
    output=handleSents(sents,pref,textid,index,sentid,annotator)
    outstring=formatList(output)
    if outfile:
        metadata=getFileNameParts(pref,textid,index,sentid)
        saveParseToFile(includeText(example,outstring),metadata, overwrite)
    handleParse(outstring,copyboard=copyboard)

def includeText(text,outstring):
    return f"# textinput = {text}\n{outstring}"

def endswithNTU(word):
    return word.endswith(NTU)

def extractNTU(glossary):
    return [entry['lemma'] for entry in filter(lambda x: endswithNTU(x['lemma']),glossary)]

def endswithSuff(suff,word):
    return word.endswith(suff)

def extractSuff(suff,glossary):
    return [entry['lemma'] for entry in filter(lambda x: endswithSuff(suff,x['lemma']),glossary)]

def inGlossary(word):
    wordlist=extractNTU(GLOSSARY)
    if word in wordlist:
        return True
    return False

def withSuff(suff,word):
    wordlist=extractSuff(suff,GLOSSARY)
    if word.lower() in wordlist:
        return True
    return False

def hasCliticAdv(token): # TODO: possibly deprecated
    if endswithNTU(token) and not inGlossary(token):
        return handleAccent(token[:-len(NTU)])
    return ''

def startswithNasal(suff):
    return suff != NTU and suff[0] in 'mn'

def hasClitic(suff,token):
    if endswithSuff(suff,token) and not withSuff(suff,token):
            nasal=startswithNasal(suff)
            return handleAccent(token[:-len(suff)],nasal)
    return ''

def mkHost(host,clitic,token,xpos=''):
    dic={}
    dic['host']=host
    dic['suff']=clitic
    if xpos:
        dic['xpos']=xpos
    dic['multiwordtoken']=token
    return dic

def extractHost(token):
    dic={}
    if token.lower() == 'maita':
        return mkHost('mayé',TA,token,'ADVRA')
    for clitic in NONHYPHEN:
        host=hasClitic(clitic,token)
        if host:
            return mkHost(host,clitic,token)
    return dic

def splitMultiWordTokens(tokens):
    MULTIWORDTOKENS.clear()
    sep=[d['lemma'] for d in AUX]
    sep.extend(CLITICS)
    newlist=[]
    for t in tokens:
        dic=extractHost(t)
        if HYPHEN in t:
            tag=''
            parts=extractTag(t)
            if parts:
                t,tag=parts
            index=t.index(HYPHEN)
            first=t[:index]
            second=t[index:]
            if second[1:] in sep:
                if second[1:] in CLITICS:
                    first=f"{first}-"
                if tag:
                    first=f"{first}/{tag}"
                newlist.extend([first,second])
            else: #TODO: has hyphen and clitic "-ntu"
                newlist.append(f"{t}/{tag}")
        elif dic: # TODO: if dic ...?
            host=dic['host']
            suff=dic['suff']
            mwt=dic['multiwordtoken']
            xpos=dic.get('xpos')
            if xpos:
                MULTIWORDTOKENS['xpos']=xpos
            MULTIWORDTOKENS[host]=mwt
            newlist.extend([host,f"{UNDERSCORE}{suff}"])
        else:
            newlist.append(t)
    return newlist

def parseSentence(sent):
    tokens=splitMultiWordTokens(tokenize(sent))
    tk=mkConlluSentence(tokens)
    return tk

def mkDict(text,pref='MooreFP1994',textid=0,sentid=1):
	groups=text.split('|')
	dic={}
	dic['yrl']=[]
	dic['yrl'].append(groups[0].strip())
	parts=[re.sub(r"\s-\s",'',part).strip() for part in re.split(r"[)(]",groups[1])]
	dic['source']=parts[1]
	dic['yrl'].append(parts[0])
	dic['por']=[]
	dic['por'].append(parts[2])
	por,eng=[s.strip() for s in re.split(r"\s-\s",groups[2])]
	dic['por'].append(por)
	dic['eng']=[]
	dic['eng'].append(eng)
	dic['eng'].append(groups[3])
	return dic

def TreebankSentences(text,pref='MooreFP1994',textid=0,index=0,sentid=1,copyboard=True, annotator=ANNOTATOR):
    output=[]
    dic=mkDict(text)
    i=0
    sentid=sentid
    index=index
    source=dic.get('source')
    yrl,eng,por=dic['yrl'],dic['eng'],dic['por']
    while(i < len(yrl)):
        output.append(
        ppText(
        [extractYrl(yrl[i]),eng[i],por[i]],
        pref,textid,index,sentid)
        )
        if source:
            output.append(f"# text_source = {source}")
        tk=parseSentence(yrl[i])
        includeAnnotator(output,annotator)
        output.append(tk.serialize())
        sentid+=1
        index+=1
        i+=1
    outstring=formatList(output)
    handleParse(outstring,copyboard=copyboard)

def writeConlluUD(sentences,outfile,pref='MooreFP1994',textid=0,sentid=1):
    i=0
    textid=textid
    sentid=sentid
    with open(outfile, 'w') as f:
        while(i < len(sentences)):
            sent=sentences[i]
            insertSentId(sent,pref,textid,sentid)
            i+=1
            sentid+=1
            print(sent.serialize(),end='',file=f)

def writeSentsConllu(sentences,outfile):
    with open(outfile, 'w') as f:
        for sent in sentences:
            print(sent.serialize(),end='',file=f)

def insertAnnotatorInSents(infile,outfile,annotator):
    sents=extractConlluSents(infile)
    insertAnnotator(sents,annotator)
    writeSentsConllu(sents,outfile)

def extractWordTagPairs(tokenlist):
	i=0
	triples=[]
	mapping={'ADV': '=adv', 'NOUN': '=n', 'ADJ': '=a',
		 'PROPN': '=p', 'VERB': '=v', 'INTJ': '=intj'}
	while(i<len(tokenlist)):
		func=''
		token=tokenlist[i]
		tokenid=token['id']
		form=token['form']
		xpos=token['xpos']
		tag=None
		if xpos:
			xpos=xpos.lower()
			tag=xpos
		upos=token['upos']
		if upos in ('INTJ','PROPN'):
			func=mapping[upos]
		feats=token['feats']
		if feats:
			rel=feats.get('Rel')
			if rel:
				tag=f"{xpos}+{rel.lower()}"
			aug=feats.get('Degree')
			if aug == 'Aug':
				func='=aug'
			aspect=feats.get('Aspect')
			if aspect in ('Freq','Hab'):
				func=f"=mkhab:x|{tag}:a|t"
			derivation=feats.get('Derivation')
			if derivation:
				if derivation == 'Priv':
					func=f"=prv:x|{xpos}"
				elif derivation == 'Coll':
					func='=col'
		misc=token['misc']
		if misc:
			spaceafter=misc.get('SpaceAfter')
			origlang=misc.get('OrigLang')
			if not func:
				if origlang:
					func=mapping.get(upos)
		if isMultitokenWord(tokenid):
			if HYPHEN in form:
				tag=tokenlist[i+1]['xpos'].lower()
			i+=2
		if func:
			tag=func
		triples.append((form,tag,spaceafter))
		i+=1
	return triples

def mkSent(triples):
	sent=""
	for form,tag,spaceafter in triples:
		sep=' '
		if spaceafter:
			sep=''
		word=f"{form}"
		if tag and tag not in ('punct','elip'):
			word=f"{form}/{tag}"
		sent=f"{sent}{word}{sep}"
	return sent

def removeLemmaAmbiguity(tk):
	i=0
	while(i<len(tk)):
		doubles=tk.filter(id=i)
		if len(doubles) == 2:
			t1=doubles[0]
			t2=doubles[1]
			form1=t1['form']
			form2=t2['form']
			if form1 == form2:
				feats1=t1.get('feats')
				feats2=t2.get('feats')
				if feats1 and feats2:
					rel1=feats1.get('Rel')
					rel2=feats2.get('Rel')
					vform1=feats1.get('VerbForm')
					vform2=feats2.get('VerbForm')
					if rel1 and rel2:
						if rel1=='Abs':
							index=tk.index(t1)
							tk.pop(index)
						elif rel2=='Abs':
							index=tk.index(t2)
							tk.pop(index)
					if vform1 and vform2:
						if vform1=='Inf':
							index=tk.index(t1)
							tk.pop(index)
						elif vform2=='Inf':
							index=tk.index(t2)
							tk.pop(index)
		i+=1

def mkTestSet(sents):
    test_sents=[]
    i=0
    while(i<len(sents)):
        tk=sents[i]
        triples=extractWordTagPairs(tk)
        sent=mkSent(triples)
        newtk=parseSentence(sent)
        removeLemmaAmbiguity(newtk)
        text=extractYrl(sent)
        newtk.metadata['sent_id']=f"{i}"
        newtk.metadata['text']=text
        newtk.metadata['sent']=sent
        test_sents.append(newtk)
        i+=1
    return test_sents


from rest_framework.decorators import api_view, permission_classes, authentication_classes
from rest_framework.permissions import IsAuthenticatedOrReadOnly, AllowAny, IsAuthenticated
from rest_framework.authentication import SessionAuthentication
from rest_framework.response import Response
from .models import User
from .serializers import UserSerializer, LoginSerializer
from rest_framework import status
from django.contrib.auth import login
# Create your views here.

@api_view(['POST'])
@permission_classes([AllowAny])
@authentication_classes([SessionAuthentication])
def registerUser(request):
    if request.method == 'POST':
        serializer = UserSerializer(data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response({'status': 200, 'payload': serializer.data, 'messge': 'your data is saved'})
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

@api_view(['POST'])
@permission_classes([AllowAny])
def loginView(request):

    serializer = LoginSerializer(data=request.data, context={ 'request': request })
    serializer.is_valid(raise_exception=True)
    user = serializer.validated_data['user']
    login(request, user)
    return Response(None, status=status.HTTP_202_ACCEPTED)

@api_view(['GET'])
@permission_classes([IsAuthenticatedOrReadOnly])
def getUsers(request):

    if request.method ==  'GET':
        user = User.objects.all()
        serializer = UserSerializer(user, many=True)
        return Response(serializer.data)

@permission_classes([IsAuthenticated])
@api_view(['GET', 'PUT', 'DELETE'])
def getUser(request, pk):

    try:
        user = User.objects.get(pk=pk)
    except User.DoesNotExist:
        return Response(status=status.HTTP_404_NOT_FOUND)

    if request.method == 'GET':
        serializer = UserSerializer(user)
        return Response(serializer.data)

    elif request.method == 'PUT':
        serializer = UserSerializer(user, data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status=status.HTTP_201_CREATED)
        return Response(serializer.data, status=status.HTTP_201_CREATED)

    elif request.method == 'DELETE':
        user.delete()
        return Response(status=status.HTTP_201_CREATED)

# user registration


import esper
import pygame
from typing import Callable, List

from src.ecs.components.c_surface import CSurface
from src.ecs.components.c_transform import CTransform
from src.ecs.components.tags.c_tag_bullet import CTagBullet
from src.ecs.components.tags.c_tag_enemie import CTagEnemie

def system_collision_bullet_enemy(world:esper.World, explosion_dict:dict, create_explosion:Callable[[esper.World, dict, pygame.Vector2], None], ene_ent:List[int]):
    bullets = world.get_components(CSurface, CTransform, CTagBullet)
    enemies = world.get_components(CSurface, CTransform, CTagEnemie)
    cb_s:CSurface
    cb_t:CTransform
    ce_s:CSurface
    ce_t:CTransform
    points = 0
    for bullet_entity, (cb_s, cb_t, c_b) in bullets:
        bull_rect = CSurface.get_area_relative(cb_s.area, cb_t.pos)
        for enemy_entety, (ce_s, ce_t, c_e) in enemies:
            enemy_rect = CSurface.get_area_relative(ce_s.area, ce_t.pos)
            if bull_rect.colliderect(enemy_rect):
                points += c_e.puntos
                world.delete_entity(bullet_entity)
                world.delete_entity(enemy_entety)
                ene_ent.remove(enemy_entety)
                create_explosion(world, explosion_dict, ce_t.pos)
    return points


import itertools
import warnings

import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
import seaborn as sns

from scipy import stats
import statsmodels.api as sm
import statsmodels.formula.api as smf
import pmdarima as pmd

import rdatasets

from tqdm import tqdm

plt.rc('axes', titlesize='medium')
plt.rc('axes', titlelocation='left')
plt.rc('axes.spines', right=False)
plt.rc('axes.spines', top=False)
sizets = (8,4.5)
plt.rc('figure', figsize=sizets)

def summarize(gb, f):
    """Summarize grouped things."""
    return gb.apply(lambda x: pd.Series(f(x)))

def compute(df, f):
    """Compute new (or replacement) columns."""
    newdf = pd.DataFrame(f(df), index=df.index)
    dropcols = [col for col in newdf.columns if col in df.columns]
    if dropcols:
        df = df.drop(columns=dropcols)
    return df.join(newdf)

def set_freq(df, freq=None):
    """Set frequency of DateTimeIndex."""
    if freq is None:
        freq = pd.infer_freq(df.index)
    return df.asfreq(freq)

def extend_timeseries(df, tmax=None, tmin=None, dt=None):
    """Extend timeseries data to later or earlier times."""
    freq = df.index.freq
    if tmax is tmin is dt is None:
        dt = 1
    if tmin is None:
        tmin = df.index.min()
    if tmax is None:
        tmax = df.index.max()
    if dt is not None:
        if isinstance(dt, int):
            if dt > 0:
                tmax += dt * freq
            elif dt < 0:
                tmin += dt * freq
        else:
            dt = pd.to_timedelta(dt)
            if dt > pd.to_timedelta('0d'):
                tmax += dt
            else:
                tmin -= dt
    index = pd.date_range(tmin, tmax, freq=freq)
    return df.reindex(index)

def suptitle(fig, text=None, **kw):
    """Add a nice left-aligned suptitle."""
    if text is None:
        fig, text = plt.gcf(), fig
    fig = fig.figure or fig
    fig.text(fig.subplotpars.left, .99, text, ha='left', va='top', size='large', **kw)

def rlabel(ax, label=None, **kw):
    """Add a right-side axis title."""
    if label is None:
        ax, label = plt.gca(), ax
    bbox = kw.pop('bbox', dict(facecolor='.9', alpha=0.2))
    ax.text(1, .5, label,
            rotation=-90, ha='left', va='center', transform=ax.transAxes,
            bbox=bbox, **kw)

def xdate(ax, fmt=None, freq=None):
    """Tweak x-axis date formatting."""
    dates = plt.matplotlib.dates
    if fmt is None:
        ax, fmt = plt.gca(), ax
    if freq:
        t1, t2 = dates.num2date(ax.get_xticks()[[0,-1]])
        ticks = pd.date_range(t1, t2, freq=freq)
        ax.set_xticks(ticks)
    ax.xaxis.set_major_formatter(dates.DateFormatter(fmt))

def plot_tsresiduals(Y, y, acf_lags=np.r_[1:26]):
    """Plot timeseries residuals for ground truth Y and estimate y."""
    fig = plt.figure()
    gs = plt.GridSpec(3, 2, figure=fig)
    ts_ax = fig.add_subplot(gs[0,:])
    axs = np.array([ts_ax] + [fig.add_subplot(gs[i,j]) for j in (0,1) for i in (1,2)])
    ax, rax, hax, acfax, pacfax = axs
    #((ax, hax), (rax, acfax)) = axs
    mask = ~(np.isnan(Y) | np.isnan(y))
    Y, y = Y[mask], y[mask]
    #dy = y - Y
    # I was surprised by this convention but ok
    dy = Y - y
    ax.plot(Y, color='k')
    ax.plot(y)
    ax.set(title='Time Series')
    lim = 1.1 * max(-dy.min(), dy.max())
    lim = -lim, lim
    rax.plot(dy)
    rax.set(ylim=lim, title='Residuals')
    sns.distplot(dy, bins=np.linspace(lim[0], lim[1], 22),
                 hist=True, kde=True, rug=True, ax=hax)
    hax.set(title='Residual Distribution')
    sm.graphics.tsa.plot_acf(dy, lags=acf_lags, ax=acfax)
    sm.graphics.tsa.plot_pacf(dy, lags=acf_lags, ax=pacfax)
    for a in axs.ravel():
        a.grid()
    plt.tight_layout()
    return fig, axs

def RMSE(Y, y):
    """Root-mean-square error."""
    return np.sqrt(np.mean((Y-y)**2))
def MAE(Y, y):
    """Mean absolute error."""
    return np.mean(np.abs(Y-y))
def MAPE(Y, y):
    """Mean absolute percent error."""
    return 100 * np.mean(np.abs((Y-y)/Y))
def MASE(Y, y):
    """TODO"""
    return np.nan # TODO

def tsaccuracy(Ytest, models):
    """Gather some metrics for a few models."""
    fs = RMSE, MAE, MAPE, MASE
    return pd.DataFrame({
        label: [ f(Ytest, model.predict(Ytest.index.min(), Ytest.index.max()))
                for f in (RMSE, MAE, MAPE, MASE) ]
        for (label, model) in models.items()
    }, index=[f.__name__ for f in fs]).T

def ciclean(ci_df):
    """Clean up conf_int() result column names."""
    ci_df = ci_df.copy()
    ci_df.columns = 'lower', 'upper'
    return ci_df

legend_right = dict(loc='center left', bbox_to_anchor=[1, .5])


import scanpy as sc
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import pandas as pd

import os
from datetime import date

def get_counts(data):
    # number of counts per spot
    data.obs['n_counts'] = data.X.sum(1)
    # number of genes per spot
    data.obs['n_genes'] = (data.X > 0).sum(1)

    data.obs['mean_counts'] = data.obs['n_counts'] / data.obs['n_genes']

    return data

def get_mean_std(data):
    np.std(data.X, axis=1)

def add_disease(data, disease_labels):
    data = data.copy()
    data.obs['disease'] = 'Unknown'
    data.obs['disease'] = data.obs['disease'].astype('<U16')
    for spot_label in disease_labels:
        m_spots = data.obs[spot_label] == 1
        data.obs['disease'][m_spots] = spot_label
    data.obs['disease'] = data.obs['disease'].astype('string').astype('category')

    return data

def plot(data, save_folder, key):
    df = data.obs

    # Plots
    # get mean counts of spots
    _, ax = plt.subplots(figsize=(12, 6))
    ax.set_yscale("log")
    ax = sns.boxplot(x="sample", y="mean_counts", data=df, ax=ax, fliersize=1, linewidth=1, hue="disease")
    ax.set_xticklabels(ax.get_xticklabels(), rotation=90)
    ax.set_xlabel("Samples")
    ax.set_ylabel("Mean UMI-counts per spot")
    ax.spines["top"].set_visible(False)
    ax.spines["right"].set_visible(False)
    # Put the legend out of the figure
    plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
    plt.tight_layout()
    plt.savefig(os.path.join(save_folder, "{}_{}".format(key, "Mean_UMI_counts_per_spot.png")), dpi=300)
    plt.close()

    # mean counts per sample
    _, ax = plt.subplots(figsize=(12, 6))
    ax.set_yscale("log")
    ax = sns.boxplot(x="sample", y="n_counts", data=df, ax=ax, fliersize=1, linewidth=1, hue="disease")
    ax.set_xticklabels(ax.get_xticklabels(), rotation=90)
    ax.set_xlabel("Samples")
    ax.set_ylabel("Total UMI-counts per spot")
    ax.spines["top"].set_visible(False)
    ax.spines["right"].set_visible(False)
    # Put the legend out of the figure
    plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
    plt.tight_layout()
    plt.savefig(os.path.join(save_folder, "{}_{}".format(key, "Total_UMI_counts_per_spot.png")), dpi=300)
    plt.close()

def main(save_folder):
    adata_wo_4_7 = sc.read(
        '/Users/christina.hillig/Documents/Projects/annData_objects/spatial/2021-07-29/Spatial Transcriptomics_unpp.h5')

    print("Dimension of samples without 4 and 7", adata_wo_4_7.shape)

    # rename LICHEN to LP
    adata_wo_4_7.obs = adata_wo_4_7.obs.rename(columns={"LICHEN": "LP"})
    adata_wo_4_7 = get_counts(data=adata_wo_4_7)
    adata_wo_4_7 = add_disease(data=adata_wo_4_7, disease_labels=['PSO', 'AE', 'LP', 'PRP'])

    # per spot
    df_spot = pd.DataFrame(columns=['sample', 'mean', 'var', 'std'])
    df_spot.loc[:, 'sample'] = adata_wo_4_7.obs['sample'].values
    nan_data = adata_wo_4_7.X.copy()
    # 1. Set zero counts to nan
    nan_data[nan_data == 0] = np.nan
    # 2. Calculate mean, std, and var
    df_spot.loc[:, 'mean'] = np.nanmean(nan_data, axis=1)
    df_spot.loc[:, 'var'] = np.nanvar(nan_data, axis=1)
    df_spot.loc[:, 'std'] = np.nanstd(nan_data, axis=1)

    del nan_data

    # save in .csv file
    df_spot.to_csv(os.path.join(save_folder, "Statistics_per_spot_wo_4_7.csv"))
    df_spot.to_excel(os.path.join(save_folder, "Statistics_per_spot_wo_4_7.xlsx"))

    # TODO per sample
    df_sample = pd.DataFrame(
        columns=['sample', 'min_count_spot', 'max_count_spot', 'mean_count_spot', 'var_count_spot', 'std_count_spot',
                 'median_counts_spot', 'total_genes_detected', 'median_genes_spot'])
    # 1. Sum up counts per sample
    for ind, sample_id in enumerate(adata_wo_4_7.obs['sample'].cat.categories):
        df_sample.loc[ind, 'sample'] = sample_id

        mask_sample = adata_wo_4_7.obs['sample'] == sample_id

        matrix_sample = adata_wo_4_7.X[mask_sample, :].copy()

        sum_over_spots = np.sum(matrix_sample, axis=0)
        sum_over_spots[sum_over_spots > 0] = 1
        df_sample.loc[ind, 'total_genes_detected'] = sum_over_spots.sum()

        #  Calculate mean, std, and var
        # 1. Get counts per sample
        counts_per_sample = np.sum(matrix_sample, axis=0)

        # 2. Set zero counts to nan
        counts_per_sample[counts_per_sample == 0] = np.nan

        # Min and Max count pro sample
        df_sample.loc[ind, 'min_count_spot'] = np.nanmin(counts_per_sample)
        df_sample.loc[ind, 'max_count_spot'] = np.nanmax(counts_per_sample)

        # 3. Calculate mean, std, and var
        mean_count_gene_sample = np.nanmean(counts_per_sample)
        df_sample.loc[ind, 'mean_count_spot'] = mean_count_gene_sample

        var_count_gene_sample = np.nanvar(counts_per_sample)
        df_sample.loc[ind, 'var_count_spot'] = var_count_gene_sample

        std_count_gene_sample = np.nanstd(counts_per_sample)
        df_sample.loc[ind, 'std_count_spot'] = std_count_gene_sample

        # Median UMI Counts per Spot
        sum_over_genes = np.sum(matrix_sample, axis=1)
        median_counts_sample = np.median(sum_over_genes)
        df_sample.loc[ind, 'median_counts_spot'] = median_counts_sample

        # Median Genes per Spot
        matrix_sample[matrix_sample > 0] = 1
        num_genes_spot_sample = np.sum(matrix_sample, axis=1, dtype='int64')
        median_genes_sample = np.median(num_genes_spot_sample)
        df_sample.loc[ind, 'median_genes_spot'] = median_genes_sample

        np.sum(matrix_sample, axis=1, dtype='int64')

    df_sample.to_csv(os.path.join(save_folder, "Statistics_per_sample_wo_4_7.csv"))
    df_sample.to_excel(os.path.join(save_folder, "Statistics_per_sample_wo_4_7.xlsx"))

    plot(data=adata_wo_4_7, save_folder=save_folder, key='wo_4_7')

    del adata_wo_4_7

    adata = sc.read(
        '/Users/christina.hillig/Documents/Projects/annData_objects/spatial/2021-03-19/adata_P15509_P16357_unpp.h5')

    print("Dimension of all samples", adata.shape)

    adata.obs = adata.obs.rename(columns={"LICHEN": "LP"})
    adata = add_disease(data=adata, disease_labels=['PSO', 'AE', 'LP', 'PRP'])
    adata = get_counts(data=adata)
    plot(data=adata, save_folder=save_folder, key='all')

if __name__ == '__main__':
    today = date.today()
    path = os.path.join("..", "..")
    # create saving folder in current project path
    savepath = os.path.join(path, "output", "UMI-counts_per_spots", str(today))
    os.makedirs(savepath, exist_ok=True)

    main(save_folder=savepath)


import itertools
from functools import reduce
print()

with open('input.txt', 'r') as f:
  raw = [l.strip()for l in f.readlines()]
input_busses = [int(b) if b != 'x' else b for b in raw[1].split(',')]

def normalize(busses, start):
  return ((((start // b) + 1) * b) - start if b != 'x' else 'x' for b in busses)

def part1(busses):
  busses = [b for b in busses if b != 'x']
  start = int(raw[0])
  normalized_departures = list(normalize(busses, start))
  wait = min(normalized_departures)
  bus_id = busses[normalized_departures.index(wait)]
  print('part1', bus_id, wait, bus_id * wait)

part1(input_busses)

# https://rosettacode.org/wiki/Chinese_remainder_theorem#Python

def chinese_remainder(n, a):
  total = 0
  prod = reduce(lambda a, b: a*b, n)
  for n_i, a_i in zip(n, a):
    p = prod // n_i
    total += a_i * mul_inv(p, n_i) * p
  return total % prod

def mul_inv(a, b):
  b0 = b
  x0, x1 = 0, 1
  if b == 1:
    return 1
  while a > 1:
    q = a // b
    a, b = b, a % b
    x0, x1 = x1 - q * x0, x0
  if x1 < 0:
    x1 += b0
  return x1

def part2(busses):
  """Prints busses such that wolframalpha can take it as input.

  e.g.:
  (x mod 7 = 0), (x mod 13 = 1), (x mod 59 = 4), (x mod 31 = 6), (x mod 19 = 7)
  """
  goal = [i for i, b in enumerate(busses) if b != 'x']
  cleaned = [b for b in busses if b != 'x']
  answer = ', '.join([f'(x mod {c} = {g})' for c, g in zip(cleaned, goal)])
  r = chinese_remainder(cleaned, goal)
  p = reduce(lambda a, b: a * b, cleaned)
  return cleaned, goal, p - r

with open('solutions.txt', 'r') as f:
  raw = [l.strip() for l in f.readlines()]
  solutions = []
  for r in raw:
    raw_problem, raw_answer = r.split('=')
    answer = int(raw_answer)
    problem = [int(p) if p != 'x' else p for p in raw_problem.split(',')]
    solutions.append((problem, answer))

  for problem, answer in solutions:
    print(part2(problem), answer)

print(part2(input_busses))


import email
import logging
import pprint
import typing as t
from collections import Sequence
from contextlib import contextmanager

from engine.utils import InvalidFlagException, is_gmail_label

if t.TYPE_CHECKING:
    from engine.models.message import Message

logger = logging.getLogger('youps')  # type: logging.Logger

@contextmanager
def _open_rfc822(message):
    #  type: (Message) -> t.Generator[email.message.Message, None, None]
    """convert a Message into a python email.message.Message.

    Usage:
        with open_rfc822(message) as rfc_contents:

    Raises:
        RuntimeError: if the fetched content doesn't have the correct uid
        RuntimeError: if the fetched content is missing the RFC822

    Returns:
        email.message.Message: python email message representation
    """

    # check if the message is initially read
    initially_unread = message.is_unread
    try:
        response = message._imap_client.fetch(
            message._uid, ['RFC822'])  # type: t.Dict[t.AnyStr, t.Any]
        if message._uid not in response:
            raise RuntimeError('Failed to get message content')
        response = response[message._uid]

        # get the rfc data we're looking for
        if 'RFC822' not in response:
            logger.critical('%s:%s response: %s' %
                            (message.folder, message, pprint.pformat(response)))
            logger.critical("%s did not return RFC822" % message)
            raise RuntimeError("Failed to get message content")
        rfc_contents = email.message_from_string(
            response.get('RFC822'))  # type: email.message.Message
        yield rfc_contents
    finally:
        # if the message was read mark it as unread
        if initially_unread:
            logger.critical("INITIALLLY UNREAD")
            # TODO see remove_flags_gmail, marking unread doesn't actually work
            message.mark_unread()

def _get_text_from_python_message(part):
    # type: (email.message.Message) -> t.Optional[t.AnyStr]

    # don't even try to convert non text parts
    main_type = part.get_content_maintype()
    if main_type != "text":
        return None

    # get the charset used to encode the message
    charset = part.get_content_charset()

    # return decoded text if possible otherwise fall back to non
    # decoded text otherwise fail
    if charset is not None:
        return unicode(part.get_payload(decode=True), charset, "ignore")
    else:
        try:
            return unicode(part.get_payload(decode=True))
        except Exception:
            raise RuntimeError("failed to convert message to etext")

# TODO refactor this to do some kind of visitor pattern or something
# make things open to extension but closed for modification
def get_content_from_message(message, return_only_text=False):
    # type: (Message) -> None
    with _open_rfc822(message) as rfc_contents:
        text = ""
        html = ""
        extra = {}

        # walk the message
        for part in rfc_contents.walk():
            # TODO respect multipart/[alternative, mixed] etc... see RFC1341
            if part.is_multipart():
                continue

            # for each part get the maintype and subtype
            sub_type = part.get_content_subtype()

            text_contents = _get_text_from_python_message(part)

            # extract plain text
            if text_contents is not None:
                text += text_contents if sub_type == "plain" else ""
                html += text_contents if sub_type == "html" else ""
            # extract calendar
            elif sub_type == "calendar":
                if sub_type not in extra:
                    extra[sub_type] = ""
                extra[sub_type] += text_contents
            # fail otherwise
            else:
                logger.critical(
                    "%s unsupported sub type %s" % (message, sub_type))
                # raise NotImplementedError(
                #     "Unsupported sub type %s" % sub_type)

        # I think this is less confusing than returning an empty string - LM
        text = text if text else None
        html = html if html else None
        # return text if we have it otherwise html
        # return text if text else html
        if return_only_text:
            return text
        else:
            extra['text'] = text
            extra['html'] = html

            return extra

def _flag_change_helper(message, uids, flags, gmail_label_func, imap_flag_func):
    # type: (Message, t.List[int], t.List[str], t.Callable[[t.List[int], t.List[str]]])
    import pprint
    flags = _check_flags(message, flags)
    if message._imap_account.is_gmail:
        # gmail_labels = filter(is_gmail_label, flags)
        # returned_labels = gmail_label_func(uids, gmail_labels)
        # logger.debug("flag change returned labels {flags}".format(
        #     flags=pprint.pformat(returned_labels)))

        # Wouldn't Gmail users would want to use gmail labels which is visible in their interface?
        not_gmail_labels = filter(lambda f: not is_gmail_label(f), flags)
        logger.exception(not_gmail_labels)
        returned_flags = gmail_label_func(uids, not_gmail_labels)
        logger.exception("flag change returned flags {flags}".format(
            flags=pprint.pformat(returned_flags)))
    else:
        returned_flags = imap_flag_func(uids, flags)
        logger.debug("flag change returned flags {flags}".format(
            flags=pprint.pformat(returned_flags)))

def _check_flags(message, flags):
    # type: (Message, t.Iterable[t.AnyStr]) -> t.Iterable[t.AnyStr]
    """Check user defined flags

    Raises:
        InvalidFlagException: The flags are invalid

    Returns:
        t.List[t.AnyStr]: the valid flags as a list
    """

    # allow user to pass in a string
    if isinstance(flags, basestring):
        flags = [flags]
    elif not isinstance(flags, Sequence):
        raise InvalidFlagException(
            "flags must be a sequence or a a string")

    if not isinstance(flags, list):
        flags = list(flags)

    # make sure all flags are strings
    for flag in flags:
        if not isinstance(flag, basestring):
            raise InvalidFlagException("each flag must be string")
    # remove extraneous white space
    flags = [flag.strip() for flag in flags]
    # remove empty strings
    flags = [flag for flag in flags if flag]
    if not flags:
        raise InvalidFlagException(
            "No valid flags. Check if flags are empty strings")
    return flags

def _save_flags(message, flags):
    # type: (Message, t.List[t.AnyStr]) -> None
    """Save new flags to the database

    removed the setter from flags since it was too dangerous.
    """
    if not message._is_simulate:
        message._schema.flags = flags
        message._schema.save()

    message._flags = flags


from flask_wtf import FlaskForm
from wtforms import StringField, SubmitField, SelectField, BooleanField
from wtforms.validators import DataRequired, ValidationError

class SubmitBusinessForm(FlaskForm):
    name = StringField("Name", validators=[DataRequired()])
    website = StringField("Website", validators=[DataRequired()])
    category = SelectField("Category", choices=[("Restaurants", "Restaurants"), ("Cafes", "Cafes"), ("Specialty Goods", "Specialty Goods"), ("Apparel", "Apparel"), ("Lifestyle", "Lifestyle")])
    address = StringField("Address", validators=[DataRequired()])
    country = StringField("Country", validators=[DataRequired()])
    submit = SubmitField("Submit")

class FilterForm(FlaskForm):
    restaurants = BooleanField("Restaurants")
    cafes = BooleanField("Cafes")
    specialty_goods = BooleanField("Specialty Goods")
    apparel = BooleanField("Apparel")
    lifestyle = BooleanField("Lifestyle")
    submit = SubmitField("Submit")


import predict
from google.cloud import aiplatform
import google.auth
import numpy as np
import streamlit as st
import pandas as pd
import matplotlib.pyplot as plt

creds,proj_id = google.auth.default()
aiplatform.init(credentials=creds,project=proj_id)

st.set_page_config(layout='centered')
st.title("What Crop Should You Plant?")
st.write("This model will recommend the most suitable crop to plant based on soil and environmental factors.  This can provide information valuable to setting a farming strategy for optimizing yields. The model is trained on Indian agriculture data.")

col1,col2  = st.columns([2,2])

with col1:
    N = st.number_input("Nitrogen", 1,150)
    P = st.number_input("Phosporus", 1,150)
    K = st.number_input("Potassium", 1,225)
    temp = st.number_input("Temperature in Celcius",0.0,50.0)

with col2:
    humidity = st.number_input("Humidity in %", 0.0,100.0)
    ph = st.number_input("Soil pH", 0.0,14.0)
    rainfall = st.number_input("Rainfall in mm",0.0,300.0)

model_input = [N,P,K,temp,humidity,ph,rainfall]

crop_list = ['Watermelon','Rice','Pomegranate','Pigeon Peas','Papaya','Orange','Muskmelon','Mung Beans','Moth Beans','Mango','Maize','Lentil','Kidney Beans','Jute','Grapes','Cotton','Coffee','Coconut','Chickpea','Blackgram','Banana','Apple']

if st.button('Predict'):
    probs = predict.predict_custom_trained_model_sample(
        # project="717860154136",
        # endpoint_id="3838513839821815808",
        project="554698915331",
        endpoint_id="6625397584234020864",
        location="us-central1",
        instances=[model_input]
    )
    crop=crop_list[np.argmax(probs)]
    html_str = f"""

The best crop for you is {crop}

    """
    st.markdown(html_str, unsafe_allow_html=True,)

    fig, ax = plt.subplots()
    ax.bar(x=crop_list,height=probs)
    ax.tick_params(labelrotation=90)
    ax.set_xlabel("Crop")
    ax.set_ylabel("Recommendation Confidence")
    st.pyplot(fig)