from .. import *
from ..nlp import *

#from jstor_xml import *
#from intext import *

from nltk import sent_tokenize
from lxml.etree import _ElementTree as ElementTree
from lxml import etree
recovering_parser = etree.XMLParser(recover=True)

from zipfile import ZipFile

if False:
    # looks like I don't use this?
    import spacy

    nlp = spacy.load('en')
    import neuralcoref

    neuralcoref.add_to_pipe(nlp)

ALL_UNIVERSITIES = {
    x[1].upper().strip()
    for x in creader(
        Path(__file__).parent.parent.joinpath("external-data", "world-universities.csv")
            .open('r', encoding='UTF8')
    )
}
max_uni_length = max( len(x) for x in ALL_UNIVERSITIES )

class ParseError(Exception):
    pass

def clean_metadata( doi, metadata_str ):

    try:
        metaXml = etree.fromstring(metadata_str, parser=recovering_parser)
    except OSError:
        raise ParseError("NO METADATA!", doi)

    def findAndCombine(query):
        return ";".join([" ".join(x.itertext()) for x in metaXml.findall(query)]).strip()

    metadata = {}
    metadata['type'] = metaXml.get("article-type")

    metadata['doi'] = doi
    # print(doi)

    metadata['title'] = findAndCombine(".//article-title")
    metadata['journal'] = findAndCombine(".//journal-title")
    metadata['publisher'] = findAndCombine(".//publisher-name")

    metadata['abstract'] = findAndCombine(".//article-meta//abstract")

    auth = []
    for group in metaXml.findall(".//contrib"):
        myname = " ".join(x.strip() for x in group.itertext())
        myname = " ".join(re.split("\s+", myname)).strip()
        auth.append(myname)
    metadata['authors'] = auth

    if len(auth) == 0 and False:
        print(doi)
        print(auth)
        print(metadata['title'])

    metadata['month'] = findAndCombine(".//article-meta//month")

    metadata['year'] = findAndCombine(".//article-meta//year")
    if ";" in metadata['year']:
        metadata['year'] = metadata['year'].split(";")[0]
    try:
        metadata['year'] = int(metadata['year'])
    except:
        raise ParseError("No valid year found")

    metadata['volume'] = findAndCombine(".//article-meta//volume")
    metadata['issue'] = findAndCombine(".//article-meta//issue")

    metadata['fpage'] = findAndCombine(".//article-meta//fpage")
    metadata['lpage'] = findAndCombine(".//article-meta//lpage")

    return metadata

def basic_ocr_cleaning(x):
    # remove multiple spaces in a row
    x = re.sub(r" +", ' ', str(x))
    # remove hyphenations [NOTE this should be updated, with respect to header and footer across pages...]
    x = re.sub(r"([A-Za-z]+)-\s+([A-Za-z]+)", "\g<1>\g<2>", x)

    x = x.strip()
    return x

def get_content_string(ocr_string):
    docXml = etree.fromstring(ocr_string, parser=recovering_parser)
    pages = docXml.findall(".//page")

    page_strings = []
    for p in pages:
        if p.text is None:
            continue
        page_strings.append(p.text)

    secs = docXml.findall(".//sec")

    for s in secs:
        if s.text is None:
            continue
        if s.text.strip() == '':
            try_another = etree.tostring(s, encoding='utf8', method='text').decode("utf8").strip()
            #print(try_another)
            if try_another == '':
                continue

            page_strings.append(try_another)
        else:
            page_strings.append(s.text.strip())

    return basic_ocr_cleaning( "\n\n".join(page_strings) )

class Sentence:
    def __init__(self, content=None ,lines=None, words=None):
        self.lines = lines
        self.content = content
        self.words = words
        self.type = None

        if self.words is not None:
            self.content = untokenize(self.words)

        # process hyphenation
        self.content = fix_paragraph(self.content)

    def parse(self):

        self.doc = nlp(self.content)
        self.root_verb = [word for word in self.doc if word.head == word ][0]

        ".+ is one thing and .+ is quite another"
        ".+ is one thing and .+ is another"
        ".+ is the same as .+"

        if str(self.root_verb) == "is":

            nsubj = follow_expand( self.root_verb, 'nsubj' )
            attr = follow_expand( self.root_verb, 'attr' )
            advc = follow_expand( self.root_verb, 'advcl' )

            if len(nsubj) == 1 and len(attr) == 1:
                self.type = "identification"
                self.args = [
                    nsubj[0],
                    attr[0],
                    advc
                ]
                print("X(%s) IS Y(%s)" % tuple(self.args[:2]))

    def __str__(self):
        return self.__repr__()

    def __repr__(self):
        return self.content

class Line:
    def __init__(self, content=None, page=None):
        self.content = content.strip()
        self.page = None
        self.type = None

        #if len(self.content) < 5:
        #    print(self.content)

        if self.content == "":
            self.type = "empty"

        try:
            numerical_content = int(self.content)
            self.type = "page_number"
            self.value = numerical_content
        except ValueError:
            pass

    def __str__(self):
        return self.__repr__()
    def __repr__(self):
        return self.content

class Page:

    @classmethod
    def from_lines(cls, lines):
        p = Page()

        for line in lines:
            p.lines.append(Line(
                # content
                content=line,
                # and context
                page=p
            ))

        p.extract_sentences()

        return p

    @classmethod
    def from_text(cls, text):
        return Page.from_lines(re.split("[\n\r]+", text))

    def __init__(self):
        self.lines = []
        self.start_stub = None
        self.end_stub = None
        self.full_sentences = None

    def __str__(self):
        return self.__repr__()
    def __repr__(self):
        return "\n".join( map(str,self.lines) )

    def extract_sentences(self):
        from nltk import sent_tokenize, word_tokenize

        sents = [ word_tokenize(ss) for ss in sent_tokenize( str(self) ) ]

        if not len(sents):
            raise ParseError("Sentences not found")

        full_sentences = [ Sentence(words=sent_words) for sent_words in sents[1:-1] ]
        start_stub = sents[0]
        end_stub = sents[-1]

        self.start_stub = start_stub
        self.end_stub = end_stub
        self.full_sentences = full_sentences

    @property
    def content(self):
        return str(self)

romanChars = "LIXMCV"

def isRoman(x):
    if not type(x) == str:
        return False

    return all( y in romanChars for y in x.upper() )

def num_pdf2page(pdf_num, page_map):
    import roman
    beforeMe = [ x for x in page_map.items() if x[0] <= pdf_num ]
    if not len(beforeMe):
        return -1
    lastBeforeMe = max( beforeMe, key=lambda x: x[0] )

    afterMe = [x for x in page_map.items() if x[0] >= pdf_num]
    if not len(afterMe):
        return -1
    firstAfterMe = min(afterMe, key=lambda x: x[0])

    if isRoman(lastBeforeMe[1]) and isRoman(firstAfterMe[1]):
        return roman.toRoman( pdf_num - lastBeforeMe[0] + roman.fromRoman(lastBeforeMe[1].upper()) )

    elif not isRoman(lastBeforeMe[1]) and not isRoman(firstAfterMe[1]):
        return pdf_num - lastBeforeMe[0] + int(lastBeforeMe[1])

    else:
        return -1

class Document:
    def __init__(self, metadata):
        self.pages = []
        self.metadata = metadata

        self.type = self.metadata['type']

        lt = self['title'].lower()
        if lt in ["book review", 'review essay']:
            self.type = 'book-review'
        if "Commentary and Debate".lower() == lt or 'Comment on'.lower() in lt:
            self.type = 'commentary'
        if lt.find('reply to') == 0:
            self.type = 'commentary'
        if 'Erratum'.lower() in lt:
            self.type = 'erratum'

    @classmethod
    def from_db(cls, _id):
        raise Exception("Haven't done yet!")

    @classmethod
    def from_xml_strings(cls, meta_xml, content_xml, doi=None, complex_parsing=True):

        metadata = cls.parse_metadata(meta_xml)
        metadata['doi'] = doi
        page_strings = cls.parse_content(content_xml)

        page_strings = list(map(basic_ocr_cleaning, page_strings))
        page_strings = [x for x in page_strings if x != ""]
        if not len(page_strings):
            raise ParseError("Empty document")

        return Document.from_pages(page_strings, metadata=metadata, complex_parsing=complex_parsing)

    @classmethod
    def from_file(cls, fn, complex_parsing=True):

        my_name = ".".join(fn.split(".")[:-1])
        doi = my_name.split("-")[-1].replace("_", "/")

        metadataFn = "%s.xml" % my_name
        metadataFn = join(dirname(metadataFn), "..", "metadata", metadataFn)

        metadata = cls.parse_metadata(open(metadataFn).read())

        page_strings = cls.parse_content(open(fn).read())

        page_strings = list(map(basic_ocr_cleaning, page_strings))
        page_strings = [x for x in page_strings if x != ""]
        if not len(page_strings):
            raise ParseError("Empty document")

        return Document.from_pages(page_strings, metadata=metadata, complex_parsing=complex_parsing)

    @classmethod
    def from_pages(cls, page_strings, metadata={}, complex_parsing=True):
        d = Document(metadata)

        if not len(page_strings):
            raise ParseError("No pages...")

        d.pages = [Page.from_lines(re.split(r"[\n\r]+", page)) for page in page_strings]

        if complex_parsing:
            if d.type == 'research-article':
                try:
                    d.find_bibliography()
                    d.extract_headers_footers()
                    #d.parse_bibliography()
                except ParseError:
                    print("Couldn't extract bib and headers from ", d['doi'])
                    raise

        return d

    @property
    def sentences(self):
        for i, p in enumerate(self.pages):
            if i > 0:
                yield Sentence(words=self.pages[i - 1].end_stub + self.pages[i].start_stub)

            p.extract_sentences()
            for s in p.full_sentences:
                yield s

    @classmethod
    def parse_content(cls, xml_string):

        import xml.etree.ElementTree as ET
        docXml = ET.ElementTree(ET.fromstring(xml_string, parser=recovering_parser))

        pages = docXml.findall(".//page")

        page_strings = []
        for p in pages:
            if p.text is None:
                continue
            page_strings.append(p.text)

        secs = docXml.findall(".//sec")

        for s in secs:
            if s.text is None:
                continue
            if s.text.strip() == '':
                try_another = etree.tostring(s, encoding='utf8', method='text').decode("utf8").strip()
                # print(try_another)
                if try_another == '':
                    continue

                page_strings.append(try_another)
            else:
                page_strings.append(s.text.strip())

        return page_strings

    @classmethod
    def parse_metadata(cls, xml_string):

        try:
            import xml.etree.ElementTree as ET
            metaXml = ET.ElementTree(ET.fromstring(xml_string, parser=recovering_parser))
        except OSError:
            raise ParseError("NO METADATA!", fn)

        assert (isinstance(metaXml, ET.ElementTree))

        def findAndCombine(query):
            return ";".join([" ".join(x.itertext()) for x in metaXml.findall(query)]).strip()

        metadata = {}
        metadata['type'] = metaXml.getroot().get("article-type")

        # print(doi)

        metadata['title'] = findAndCombine(".//article-title")
        metadata['journal'] = findAndCombine(".//journal-title")
        metadata['publisher'] = findAndCombine(".//publisher-name")

        metadata['abstract'] = findAndCombine(".//article-meta//abstract")

        auth = []
        for group in metaXml.findall(".//contrib"):
            myname = " ".join(x.strip() for x in group.itertext())
            myname = " ".join(re.split("\s+", myname)).strip()
            auth.append(myname)
        metadata['authors'] = auth

        if len(auth) == 0 and False:
            print('no authors...')
            print(auth)
            print(metadata['title'])

        metadata['month'] = findAndCombine(".//article-meta//month")

        metadata['year'] = findAndCombine(".//article-meta//year")
        if ";" in metadata['year']:
            metadata['year'] = metadata['year'].split(";")[0]
        try:
            metadata['year'] = int(metadata['year'])
        except:
            raise ParseError("No valid year found")

        metadata['volume'] = findAndCombine(".//article-meta//volume")
        metadata['issue'] = findAndCombine(".//article-meta//issue")

        metadata['fpage'] = findAndCombine(".//article-meta//fpage")
        metadata['lpage'] = findAndCombine(".//article-meta//lpage")

        return metadata

    @property
    def num_pages(self):
        return len(self.pages)

    def apply_page_map(self, page_map):
        self.page_map = page_map

        for i, p in enumerate(self.pages):
            p.page_mapped = num_pdf2page(pdf_num=i, page_map=self.page_map)

    def eliminate_metadata(self):
        from fuzzysearch import find_near_matches

        if self.type != 'research-article':
            print("Unimplemented: extracting headers and footers on non-research-article")
            return

        if not len(self.pages):
            raise ParseError()

        fp = str(self.pages[0])
        fp = fp.upper()

        # extracting title, heading stuff on first page, and abstract
        to_extract_from_first_page = [
            self['title']
        ]

        if len(self['abstract']):
            to_extract_from_first_page.append(self['abstract'])

        for a in self['authors']:
            arev = a.split()
            arev = arev[-1] + ", " + " ".join(arev[:-1])

            to_extract_from_first_page += [a, arev]

        found = []

        # to get the university, not included in most metadatas...
        if False:
            found_uni = []
            for u in ALL_UNIVERSITIES:
                found.append((fp.find(u), len(u)))
        print(to_extract_from_first_page)

        for word in to_extract_from_first_page:
            for match in find_near_matches(word.upper(), fp, max_l_dist=len(word) // 10):
                # match = word[match.start:match.end]
                # print('match {}'.format(match))
                # index = ls.find(match)
                print(match.start, match.end, match.matched, match.dist)
                found.append( (match.start, match.end-match.start) )

        # limit to those strings which were actually found
        found = list(filter(lambda x: x[0] >= 0, found))

        if len(found):
            # these strings should be "end to end", i.e. less than 5 characters separating them
            found_ends = [x[0] + x[1] for x in found]
            found_min_beginning = min(x[0] for x in found)
            found = list(
                filter(lambda x: any(abs(x[0] - y) < 5 for y in found_ends) or x[0] == found_min_beginning, found)
            )

            # we then cut after these metadata fields!
            doc_start = max(found, key=lambda x: x[0] + x[1])
            doc_start = doc_start[0] + doc_start[1]

            newfp_text = str(self.pages[0])[doc_start:].strip()
            # newfp = Page.from_text(newfp_text)
            self.pages[0] = newfp_text
            #print("Trimmed woohoo: %s\n\n%s"%(fp, newfp_text))

    def extract_headers_footers(self):
        # extracting headers via voting
        num_identical_necessary = len(self.pages) / 4

        flines = [str(x).split()[:20] for x in self.pages]
        grams = Counter(
            tuple(x[:i])
            for i in range(1, 15)
            for x in flines
        )

        candidates = set(x for x, c in grams.items() if c > num_identical_necessary)
        maximal_candidates = []
        for c in candidates:
            keep = True
            for c2 in candidates:
                if len(c) >= len(c2):
                    continue
                if c[:min(len(c), len(c2))] == c2[:min(len(c), len(c2))]:
                    keep = False

            if keep:
                maximal_candidates.append(c)

        self.headers = [" ".join(x) for x in maximal_candidates]
        for h in self.headers:

            # don't want to get rid of the title on the first page.
            # that's the job for another algorithm
            # therefore, the [1:]
            for i, p in enumerate(self.pages[1:]):
                try:
                    if str(p).index(h) < 30:
                        self.pages[i+1] = Page.from_text(str(p).replace(h, "").strip())
                except ValueError:
                    continue

    def parse_bibliography(self):
        if self.type != 'research-article':
            print("Unimplemented: parsing bibliography on non-research-article")
            self.bibliography = []
            return

        from citation_parser import citation_grammars
        from citation_parser import CitationVisitor
        from parsimonious.exceptions import ParseError
        cv = CitationVisitor()

        def name_split(x):

            # go backwards, finding the last colon or number. then forwards to the next period.
            parts = re.split(r"([:][^:.]*\.|[0-9][^0-9.]*\.)", x)
            parts = ["".join(parts[:-1]), parts[-1]]

            return parts

        t = self.bibString
        # split by years first, to find separate citations
        yspl = re.split("((?:19|20)[0-9]{2}[^\-])", t)
        print("\n".join("--%s" % x for x in yspl))
        bib_entries = []
        for i in range(1, len(yspl), 2):
            # lands on every year.
            if i == 1:
                bib_entries.append("".join([
                    yspl[0],  # stub at beginning
                    yspl[1],  # year that was split
                    name_split(yspl[2])[0]  # split off the name from the bulk of the citation
                ]).strip())
                continue

            bib_entries.append("".join([
                name_split(yspl[i - 1])[1],  # name of the last one
                yspl[i],  # year that was split
                name_split(yspl[i + 1])[0]  # split off the name from the bulk of the citation
            ]).strip())

        print("\n".join("--%s" % x for x in bib_entries))

        errors = 0
        success = 0
        last_success = None

        parsed_bibs = []

        for bib_entry in bib_entries:
            # print("Parsing %s" % bib_entry)

            # this block of code tries to replace ------ at the beginning of the line
            # with the most recent successfull parse
            if last_success is not None:
                a = last_success['authors']
                fa = list(a[0])
                fa = fa[-1] + ", " + " ".join(fa[:-1])

                a = [" ".join(x) for x in a]

                if len(a) > 1:
                    author_string = ", ".join(
                        [fa] +
                        a[1:-1] +
                        ["and " + a[-1]]
                    )
                else:
                    author_string = fa

                bib_entry = re.sub("^[-–—]+", author_string, bib_entry)

            # now we throw it into the parser.
            # not everything works, but it does a pretty good job I think, for now.
            for cg in citation_grammars:
                try:
                    p = cg.parse(bib_entry)
                    p = cv.visit(p)
                    p = p[1][0]

                    last_success = p
                    parsed_bibs.append(p)
                    success += 1
                    break
                except ParseError:
                    errors += 1
                    last_success = None

        self.bibliography = parsed_bibs

    def find_bibliography(self):
        if self.type != 'research-article':
            print("Unimplemented: finding bibliography on non-research-article")
            return

        num = 0
        pid = None
        for pi, p in enumerate(self.pages):
            fr = re.findall("(REFERENCES|References|Literature Cited)", str(p))
            if not len(fr):
                continue
            num += len(fr)
            pid = pi

        if num > 1:
            #raise ParseError("More than one REFERENCES string!")
            self.bibString = ""
            return

        if pid is None:
            #raise ParseError("No REFERENCE string found...", "\n".join( str(p) for p in self.pages ))
            self.bibString = ""
            return

        # print("Found bibliography starting on page ", pid+1)

        bibString = []
        for i in range(pid, len(self.pages)):
            if i == pid:
                bibString.append("".join(re.split("(REFERENCES|References|Literature Cited)", str(self.pages[i]))[2:]))
                continue

            bibString.append(str(self.pages[i]))

        self.bibString = "\n".join(bibString).strip()

        newptext = str(self.pages[pid]).split("REFERENCES")[0].strip()
        if newptext == "":
            # print("Deleting page", pid+1)
            self.pages.pop()
        else:
            # print("Truncating page", pid+1)
            self.pages[pid] = Page.from_text(newptext)

        # print("Deleting pages ", pid+2, "through", len(self.pages))
        for i in range(len(self.pages) - pid):
            self.pages.pop()

    def __str__(self):
        return self.__repr__()

    def __repr__(self):
        return "\n\n".join(map(str, self.pages))

    def __getitem__(self, item):
        return self.metadata[item]

_zp_cache = {}

def doi2xml(zipdir, doi):
    zf, fn, fn2 = doi2fn(zipdir, doi)

    archive = ZipFile(zf, 'r')
    return (
        archive.read(fn),
        archive.read(fn2)
    )

def load_zd(zipdir):
        zipfiles = list(Path(zipdir).glob("*.zip"))

        zret = defaultdict(set)

        for zf in zipfiles:
            archive = ZipFile(zf, 'r')
            files = archive.namelist()

            for fn in files:
                if 'ocr/' in fn or 'ngram/' in fn:
                    continue

                fn2 = fn.replace("metadata", "ocr").replace(".xml", ".txt")

                mydoi = fn2doi(fn)
                zret[ mydoi ] = (zf, fn, fn2)

        _zp_cache[zipdir] = zret

def doi2fn(zipdir, doi):
    fn = "journal-article-%s.txt" % doi.replace("/", "_")

    if zipdir not in _zp_cache:
        load_zd(zipdir)

    return _zp_cache[zipdir][ doi ]

def fn2doi(fn):
    my_name = ".".join(fn.split(".")[:-1])
    doi = my_name.split("-")[-1].replace("_", "/")
    return doi

def _zip_read_N(zipfiles):
    all_files = []
    for zf in zipfiles:
        archive = ZipFile(zf, 'r')
        files = archive.namelist()
        names = list(set(getname(x) for x in files))

        all_files += [(archive, name) for name in names]

    return len(all_files)

def _zip_doc_iterator(zipfiles, SKIP_N=None):
    from random import shuffle

    all_files = []
    for zf in zipfiles:
        try:
            archive = ZipFile(zf, 'r')
        except:
            print("Erroring on file ", zf)
            raise

        files = archive.namelist()
        names = list(set(getname(x) for x in files))

        all_files += [(archive, name) for name in names]

    shuffle(all_files)

    for i,(archive, name) in enumerate(all_files):

        if SKIP_N is not None:
            if i % (SKIP_N+1) != 0:
                #skip_counter += 1
                continue

        try:
            yield (
                name.split("-")[-1].replace("_", "/"),
                archive.read("metadata/%s.xml" % name),
                archive.read("ocr/%s.txt" % name).decode('utf8')
            )
        except KeyError:  # some very few articles don't have both
            continue

# some types of titles should be immediately ignored
def title_looks_researchy(lt):
    lt = lt.lower()
    lt = lt.strip()

    for x in ["book review", 'review essay', 'back matter', 'front matter', 'notes for contributors',
              'publication received', 'errata:', 'erratum:']:
        if x in lt:
            return False

    for x in ["commentary and debate", 'erratum', '']:
        if x == lt:
            return False

    return True

def doc_iterator(jstor_zip_base, research_limit=True, complex_parsing=True, SKIP_N=None):
    zipfiles = list(Path(jstor_zip_base).glob("*.zip"))

    NDOC = _zip_read_N(zipfiles)

    print("Iterating over ", NDOC, "documents")

    for i, (doi, metadata_str, ocr_str) in enumerate(_zip_doc_iterator(zipfiles, SKIP_N)):

        try:
            drep = Document.parse_metadata(metadata_str)

            if research_limit:
                if drep['type'] != 'research-article':
                    continue

                lt = drep['title'].lower()
                if not title_looks_researchy(lt):
                    continue

                # Don't process the document if there are no authors
                if not len(drep['authors']):
                    continue

            d = Document.from_xml_strings(metadata_str, ocr_str, complex_parsing=complex_parsing)
            d.metadata['doi'] = doi
        except ParseError:
            continue

        drep['content'] = str(d)
        drep['doi'] = doi
        drep['citations'] = list(citations_iterator(drep))

        yield doi, drep

def citation_parens_iterator(t):
    citations = extractCitation(t)
    for c in citations:
        name = c['names']
        name = name.replace("'s", "")  # change "Simmel's" to "Simmel"

        years = c['year'].split(",")
        for y in years:
            y = y.strip()  # get rid of the extra spacing
            y = re.findall(r'[0-9]{4}', y)  # this strips out the 'a' in '1995a'
            if not len(y):
                faulty_years.add(c['year'])
                continue

            y = y[0]
            yield "%s (%s)" % (name, y)

def citations_iterator(drep):

    for index, (parenStart, parenContents) in enumerate(getOuterParens(drep['content'])):

        citations = list(citation_parens_iterator(parenContents))
        if not len(citations):
            continue

        citation = {
            "citations": citations,
            "contextLeft": drep['content'][parenStart - 400 + 1:parenStart + 1],
            "contextRight": drep['content'][
                            parenStart + len(parenContents) + 1:parenStart + len(parenContents) + 1 + 100],
            "where": parenStart
        }

        # cut off any stuff before the first space
        first_break_left = re.search(r"[\s.!?]+", citation['contextLeft'])
        if first_break_left is not None:
            clean_start_left = citation['contextLeft'][first_break_left.end():]
        else:
            clean_start_left = citation['contextLeft']

        # cut off any stuff after the last space
        last_break_right = list(re.finditer(r"[\s.!?]+", citation['contextRight']))
        if len(last_break_right):
            clean_end_right = citation['contextRight'][:last_break_right[-1].start()]
        else:
            clean_end_right = citation['contextRight']

        # we don't want anything more than a sentence

        sentence_left = sent_tokenize(clean_start_left)
        if len(sentence_left):
            sentence_left = sentence_left[-1]
        else:
            sentence_left = ""

        sentence_right = sent_tokenize(clean_end_right)[0]
        if len(sentence_right):
            sentence_right = sentence_right[0]
        else:
            sentence_right = ""

        # finally, strip the parentheses from the string
        sentence_left = sentence_left[:-1]
        sentence_right = sentence_right[1:]

        # add the thing in context
        full = sentence_left + "<CITATION>" + sentence_right

        citation['contextPure'] = sentence_left
        yield citation

def doc_iterator_forMongo(jstor_zip_base,
                 journals_filter=None,
                 debug=False):

    zipfiles = list(Path(jstor_zip_base).glob("*.zip"))

    for i, (doi, metadata_str, ocr_str) in enumerate(_zip_doc_iterator(zipfiles)):
        try:
            drep = clean_metadata(doi, metadata_str)

            # only include journals in the list "included_journals"
            if journals_filter is not None and (drep['journal'].lower() not in journals_filter):
                continue

            if debug: print("got meta")

            if drep['type'] != 'research-article':
                continue

            # some types of titles should be immediately ignored
            def title_looks_researchy(lt):
                lt = lt.lower()
                lt = lt.strip()

                for x in ["book review", 'review essay', 'back matter', 'front matter', 'notes for contributors',
                          'publication received', 'errata:', 'erratum:']:
                    if x in lt:
                        return False

                for x in ["commentary and debate", 'erratum', '']:
                    if x == lt:
                        return False

                return True

            lt = drep['title'].lower()
            if not title_looks_researchy(lt):
                continue

            # Don't process the document if there are no authors
            if not len(drep['authors']):
                continue

            drep['content'] = get_content_string(ocr_str)

            drep['citations'] = list(citations_iterator(drep))

            yield doi, drep

        except ParseError as e:
            print("parse error...", e.args, doi)

class jstor_counter:

    def __init__(
            self,
            jstor_zip_base, output_database='default-database-name', name_blacklist=[],
            RUN_EVERYTHING=False, complex_parsing=False,
            groups=None, group_reps=None,
            citations_filter=None, journals_filter=None, debug=False,
            CONSOLIDATE_ITERS=0, term_whitelist = set(), NUM_TERMS_TO_KEEP=100, stopwords = None,
            SKIP_N=None
    ):

        if stopwords is None:
            # getting ready for term counting
            from nltk.corpus import stopwords as sw
            self.stopwords = set(sw.words('english'))
        else:
            self.stopwords = stopwords

        self.cits = 0
        self.last_print = 0
        self.citations_skipped = 0
        self.no_citations = 0

        self.jstor_zip_base = Path(jstor_zip_base)
        assert(self.jstor_zip_base.exists())

        self.output_database = output_database
        self.name_blacklist = name_blacklist
        self.RUN_EVERYTHING = RUN_EVERYTHING
        self.groups = groups
        self.group_reps = group_reps
        self.citations_filter = citations_filter
        self.journals_filter = journals_filter
        self.debug = debug
        self.complex_parsing = complex_parsing
        self.SKIP_N = SKIP_N

        self.CONSOLIDATE_ITERS = CONSOLIDATE_ITERS
        self.NUM_TERMS_TO_KEEP = NUM_TERMS_TO_KEEP

        # Instantiating counters
        self.ind = defaultdict(lambda: defaultdict(int))
        self.track_doc = defaultdict(lambda: defaultdict(set))
        self.doc = defaultdict(lambda: defaultdict(int))

        self.term_whitelist = term_whitelist

    def cnt(self, term, space, doc):
        if ".".join(sorted(space.split("."))) != space:
            raise Exception(space, "should be sorted...")

        # it's a set, yo
        self.track_doc[space][term].add(doc)
        # update cnt_doc
        self.doc[space][term] = len(self.track_doc[space][term])
        # update ind count
        self.ind[space][term] += 1

    def consolidate_terms(self):
        have_now = set(self.doc['t'])
        # this is where the filtering occurs

        to_keep = set()
        if True:
            # takes terms based on the maximum number I can take...
            terms = list(self.doc['t'].keys())
            counts = np.array([self.doc['t'][k] for k in terms])
            argst = list(reversed(np.argsort(counts)))

            to_keep = [terms[i] for i in argst if '-' in terms[i][0]][
                      :self.NUM_TERMS_TO_KEEP // 2]  # half should be 2-tuples
            to_keep += [terms[i] for i in argst if not '-' in terms[i][0]][
                       :self.NUM_TERMS_TO_KEEP // 2]  # half should be 1-tuples

            to_remove = have_now.difference(to_keep)
            to_remove = set("-".join(x) for x in to_remove)

        if False:
            # takes the top 5000 terms in terms of yearly count
            sort_them = sorted(self.doc['fy.t'], key=lambda x: -self.doc['fy.t'][x])
            to_keep = defaultdict(set)

            i = 0
            while not len(to_keep) or (
                    min(len(x) for x in to_keep.values()) < NPERYEAR and
                    i < len(sort_them)
            ):
                # adds the term to the year set, if it's not already "full"
                me = sort_them[i]
                me_fy, me_t = me

                # eventually, we don't count it :P
                if self.doc['t'][me_t] < CONSOLIDATION_CUTOFF:
                    break

                if len(to_keep[me_fy]) < NPERYEAR:
                    to_keep[me_fy].add(me_t)
                i += 1

            if False:  # useful for debugging
                print({
                    k: len(v)
                    for k, v in to_keep.items()
                })

            to_keep = set(chain.from_iterable(x for x in to_keep.values()))
            to_remove = have_now.difference(to_keep)

        # so that we never log counts for these again:
        self.term_whitelist.update([x[0] for x in to_keep])

        # the rest of the code is pruning all other term counts for this term in memory
        print("consolidating... removing", len(to_remove), 'e.g.', sample(to_remove, 5))

        to_prune = ['t', 'fy.t', 'fj.t', 'c.t']
        for tp in to_prune:

            whichT = tp.split(".").index('t')  # this checks where 't' is in the name of the variable (first or second?)

            print("pruning '%s'..." % tp)

            tydels = [x for x in self.doc[tp] if x[whichT] in to_remove]

            print("old size:", len(self.doc[tp]))
            for tr in tydels:
                del self.doc[tp][tr]
                del self.ind[tp][tr]
            print("new size:", len(self.doc[tp]))

        print("final terms: ", ", ".join(sample(list("-".join(list(x)) for x in self.doc['t']), 200)))

    def account_for(self, doc):

        if 'citations' not in doc or not len(doc['citations']):
            # print("No citations", doc['doi'])
            return

        for c in doc['citations']:
            if 'contextPure' not in c:
                raise Exception("no contextPure...")

            for cited in c['citations']:

                if self.citations_filter is not None and (cited not in self.citations_filter):
                    self.citations_skipped += 1
                    continue

                self.cits += 1
                self.cnt(doc['year'], 'fy', doc['doi'])

                # citation
                self.cnt(cited, 'c', doc['doi'])

                # journal
                self.cnt(doc['journal'], 'fj', doc['doi'])

                # journal year
                self.cnt((doc['journal'], doc['year']), 'fj.fy', doc['doi'])

                # citation journal
                self.cnt((cited, doc['journal']), 'c.fj', doc['doi'])

                # citation year
                self.cnt((cited, doc['year']), 'c.fy', doc['doi'])

            # constructing the tuples set :)
            sp = c['contextPure'].lower()
            sp = re.sub("[^a-zA-Z\s]+", "", sp)  # removing extraneous characters
            sp = re.sub("\s+", " ", sp)  # removing extra characters
            sp = sp.strip()
            sp = sp.split()  # splitting into words

            sp = [x for x in sp if x not in self.stopwords]  # strip stopwords

            if False:
                tups = set(zip(sp[:-1], sp[1:]))  # two-word tuples
            elif False:
                tups = set((t1, t2) for t1 in sp for t2 in sp if t1 != t2)  # every two-word pair :)
            else:

                tups = set("-".join(sorted(x)) for x in set(zip(sp[:-1], sp[1:])))  # two-word tuples
                tups.update(sp)  # one-word tuples

            # print(len(tups),c['contextPure'], "---", tups)

            if len(self.term_whitelist):
                tups = [x for x in tups if x in self.term_whitelist]

            # just term count, in case we are using the `basic` mode
            for t1 in tups:
                # term
                self.cnt((t1,), 't', doc['doi'])

                # term year
                self.cnt((doc['year'], t1), 'fy.t', doc['doi'])

            if self.RUN_EVERYTHING:

                for cited in c['citations']:

                    if self.use_included_citations_filter and (cited not in self.included_citations):
                        continue

                    # term features
                    for t1 in tups:

                        # cited work, tuple
                        self.cnt((cited, t1), 'c.t', doc['doi'])

                        # term journal
                        self.cnt((doc['journal'], t1), 'fj.t', doc['doi'])

                        if False:  # eliminating data I'm not using

                            # author loop
                            for a in doc['authors']:
                                # term author
                                self.cnt((a, t1), 'fa.t', doc['doi'])

                        if len(self.term_whitelist):  # really don't want to do this too early. wait until it's narrowed down to the 5k
                            # term term...
                            for t2 in tups:
                                # if they intersect each other, continue...
                                if len(set(t1).intersection(set(t2))) >= min(len(t1), len(t2)):
                                    continue

                                # term term
                                self.cnt((t1, t2), 't.t', doc['doi'])

                    # author loop
                    for a in doc['authors']:
                        # citation author
                        self.cnt((cited, a), 'c.fa', doc['doi'])

                        # year author journal
                        self.cnt((a, doc['journal'], doc['year']), 'fa.fj.fy', doc['doi'])

                        # author
                        self.cnt((a,), 'fa', doc['doi'])

                    # add to counters for citation-citation counts
                    for cited1 in cited['citations']:
                        for cited2 in cited['citations']:
                            if cited1 >= cited2:
                                continue

                            self.cnt((cited1, cited2), 'c.c', doc['doi'])
                            self.cnt((cited1, cited2, doc['year']), 'c.c.fy', doc['doi'])

    def count(self):

        seen = set()

        skipped = 0

        total_count = Counter()
        doc_count = Counter()
        pair_count = Counter()

        debug = False

        di = doc_iterator(self.jstor_zip_base, research_limit=True, complex_parsing=self.complex_parsing, SKIP_N=self.SKIP_N)

        #skip_counter = 0

        print(f"Will print updated statistics every {1000//(self.SKIP_N+1)} documents.")

        for i, (doi, drep) in enumerate(di): #!!!!!!!!!!!!!!!!! fix this now...

            if i % (1000//(self.SKIP_N+1)) == 0:
                print("Document", i, "...",
                      #skip_counter, "skipped...",
                      len(self.doc['fj'].keys()), "journals...",
                      len(self.doc['c'].keys()), "cited works...",
                      len(self.doc['fa'].keys()), "authors...",
                      len(self.doc['t'].keys()), "terms used...",
                      #citations_skipped, "skipped citations...",
                      self.doc['t'][('social',)], "'social' terms"
                      )

            # sometimes multiple journal names map onto the same journal, for all intents and purposes
            #if drep['journal'] in self.journjournal_map:
            #    drep['journal'] = journal_map[drep['journal']]

                # only include journals in the list "included_journals"
            if self.journals_filter is not None and (drep['journal'] not in self.journals_filter):
                continue

            # consolidating "terms" counter as I go, to limit RAM overhead
            # I'm only interested in the most common 1000
            if self.CONSOLIDATE_ITERS > 0:
                if self.cits - self.last_print > self.CONSOLIDATE_ITERS:
                    print("Citation %s" % self.cits)
                    print("Term %s" % len(self.doc['t']))
                    # print(sample(list(self.doc['t']), 10))
                    last_print = self.cits
                    self.consolidate_terms()

            if 'citations' not in drep or not len(drep['citations']):
                # print("No citations", doc['doi'])
                self.no_citations += 1
                continue

            # now that we have all the information we need,
            # we simply need to "count" this document in a few different ways
            self.account_for(drep)

    def save_counters(self):
        db = Dataset(self.output_database)
        for k, count in self.doc.items():
            varname = "doc ___ %s" % k
            db.save_variable(varname, dict(count))

        for k, count in self.ind.items():
            varname = "ind ___ %s" % k
            db.save_variable(varname, dict(count))


#1

import statistics

numbers = [0,1]
print(statistics.median_low(numbers))
print(statistics.median(numbers))

#2

def count_characters(stringlist):
    count = 0
    for i in range(len(stringlist)):
        for j in range(len(stringlist[i])):
            count += 1
    return count

print(count_characters(["You", "may", "say", "I'm", "a", "dreamer"]))

#3

def my_reverse(stringlist):
    count = 0
    newlist = []
    for i in range(len(stringlist)):
       newlist.append(stringlist[len(stringlist) - 1 - count])
       count += 1
    return newlist

print(my_reverse(["You", "may", "say", "I'm", "a", "dreamer"]))

def create_file(file_name, n):
    outfile = open(file_name, "w")
    for i in range(1, n + 1):
        outfile.write(str(i) + "\n")
    outfile.close()
create_file("jbd.txt", 5)


from math import sqrt, atan2, sin, cos, pi

from config import nodesize, ballsize

eps = 1e-4
inf = 1e9

class Vector:
    def __init__(self, x = None, y = None, begin = None, end = None):
        if x is not None and y is not None:
            self.x = x
            self.y = y
        else:
            self.x = end[0] - begin[0]
            self.y = end[1] - begin[1]

    def speed(self):
        return sqrt(self.x * self.x + self.y * self.y)

    def __add__(self, v):
        return Vector(x = self.x + v.x, y = self.y + v.y)

    def __sub__(self, v):
        return Vector(x = self.x - v.x, y = self.y - v.y)

    def __mul__(self, v):
        return Vector(x = self.x * v, y = self.y * v)

    def __truediv__(self, v):
        return Vector(x = self.x / v, y = self.y / v)

    def __str__(self):
        return str([self.x, self.y])

    def normalize(self):
        v = self.speed()
        self.x /= v
        self.y /= v

def scal(a, b):
    return a.x * b.x + a.y * b.y

def cross(a, b):
    return a.x * b.y - a.y * b.x

def degree(a, b):
    return abs(atan2(cross(a, b), scal(a, b)))

def distance(a, b):
    return sqrt((a[0] - b[0]) * (a[0] - b[0]) + (a[1] - b[1]) * (a[1] - b[1]))

def sgn(x):
    if x > 0:
        return 1
    elif x == 0:
        return 0
    elif x < 0:
        return -1

def wall_collision(x, y1, y2, ball, v):
    if sgn(x - ball[0]) != sgn(v.x):
        return None
    else:
        time = abs(x - ball[0]) / abs(v.x)
        result = [x, ball[1] + v.y * time]
        if y1 <= result[1] <= y2:
            return result
        else:
            return None

def floor_collision(y, x1, x2, ball, v):
    if v.y >= 0 or ball[1] < y:
        return None
    else:
        time = (ball[1] - y) / -v.y
        result = [ball[0] + v.x * time, y]
        if x1 <= result[0] <= x2:
            return result
        else:
            return None

def node_collision(node, ball, v):
    if abs(v.x) < eps and abs(v.y) < eps:
        return None

    dist = (nodesize + ballsize) / 2

    p = [ball[0] + v.x, ball[1] + v.y]

    for i in range(2):
        p[i] -= node[i]
        ball[i] -= node[i]

    a = p[1] - ball[1]
    b = ball[0] - p[0]
    c = p[0] * ball[1] - p[1] * ball[0]

    l = [
        -(a * c) / (a * a + b * b),
        -(b * c) / (a * a + b * b)
    ]

    if abs(distance([0, 0], l) - dist) < eps:
        return [l[0] + node[0], l[1] + node[1]]
    elif distance([0, 0], l) < dist:
        d = sqrt(dist * dist - ((c * c) / (a * a + b * b)))
        m = sqrt((d * d) / (a * a + b * b))

        res = []
        res.append([
            l[0] + b * m + node[0],
            l[1] - a * m + node[1]
        ])

        res.append([
            l[0] - b * m + node[0],
            l[1] + a * m + node[1]
        ])

        ball = [ball[0] + node[0], ball[1] + node[1]]

        if distance(ball, node) < dist - eps:
            return (min(res, key = lambda x: distance(x, ball)), True)
        else:
            return (min(res, key = lambda x: distance(x, node)), False)

    return None

def calculate_vector(v1, m1, v2, m2):
    pass


import celerite
import numpy as np

class ExpTerm(celerite.terms.Term):
    """
    A simple exponential kernel.
    """
    parameter_names = ("log_amp", "length")

    def get_real_coefficients(self, params):
        log_amp, length = params
        return (
            np.exp(log_amp), 1 / length
        )

class ExpTermLog(celerite.terms.Term):
    """
    A simple exponential kernel.
    """
    parameter_names = ("log_amp", "log10_length")

    def get_real_coefficients(self, params):
        log_amp, log10_length = params
        length = 10**log10_length
        return (
            np.exp(log_amp), 1 / length
        )

class SimpleProfileTerm(celerite.terms.Term):
    """
    This class defines the 'simple as possible' kernel, with a Gaussian in the
    phase direction and an exponential in the time direction.
    """
    parameter_names = ("log_amp", "log10_width", "log10_length")

    def __init__(self, *args, **kwargs):
        ncoef = kwargs.pop("ncoef", 32)
        super(SimpleProfileTerm, self).__init__(*args, **kwargs)
        self.ncoef = ncoef

    def get_real_coefficients(self, params):
        log_amp, log_width, log10_length = params
        width = 10 ** log_width
        conc = 10**-log10_length
        zero_coef = np.sqrt(2 * np.pi * width ** 2)
        return (
            np.exp(log_amp) * zero_coef, conc
        )

    def get_complex_coefficients(self, params):
        log_amp, log_width, log10_length = params
        width = 10 ** log_width
        conc = 10 ** -log10_length
        omega0 = 2 * np.pi

        nc = self.ncoef
        gauss_denominator = 2 * np.pi ** 2 * width ** 2
        gauss_scale = 2 * np.sqrt(2 * np.pi * width ** 2)
        n = np.arange(nc) + 1

        d = omega0 * n

        a = np.exp(log_amp) * gauss_scale * np.ones_like(n) * np.exp(-gauss_denominator * n ** 2)
        b = np.zeros_like(n)
        c = np.ones_like(n) * conc

        return (
            a, b, c, d
        )

def make_custom_profile_term(phase_kernel, time_kernel, omega0=2 * np.pi, optimise_tau_real_only=False):
    pnames = time_kernel.parameter_names + phase_kernel.parameter_names

    time_mask = np.array([k in time_kernel.parameter_names for k in pnames], dtype=bool)
    fourier_mask = np.array([k in phase_kernel.parameter_names for k in pnames], dtype=bool)

    def extract_time_args(kwargs):
        return {k: kwargs[k] for k in time_kernel.parameter_names}

    class HalfProfileTerm(celerite.terms.Term):
        parameter_names = pnames

        def __init__(self, *args, **kwargs):
            super(HalfProfileTerm, self).__init__(*args, **kwargs)
            self.timekernel = time_kernel(**extract_time_args(kwargs))

        def get_real_coefficients(self, params):
            time_params = params[time_mask]
            fourier_params = params[fourier_mask]
            p, r = self.timekernel.get_real_coefficients(time_params)
            a = p * phase_kernel.theta0(fourier_params)
            c = r
            return (a, c)

        def get_complex_coefficients(self, params):
            time_params = params[time_mask]
            fourier_params = params[fourier_mask]
            p, r = self.timekernel.get_real_coefficients(time_params)

            n, theta_n = phase_kernel(fourier_params)
            a = 2 * p * theta_n
            b = np.zeros_like(n)
            c = r * np.ones_like(n)
            d = omega0 * n

            return (
                a, b, c, d
            )

    class CmplxProfileTerm(celerite.terms.Term):
        parameter_names = pnames

        def __init__(self, *args, **kwargs):
            super(CmplxProfileTerm, self).__init__(*args, **kwargs)
            self.timekernel = time_kernel(**extract_time_args(kwargs))

        def get_complex_coefficients(self, params):
            time_params = params[time_mask]
            fourier_params = params[fourier_mask]
            p, q, r, s = self.timekernel.get_complex_coefficients(time_params)

            n, theta_n = phase_kernel(fourier_params)
            theta_0 = phase_kernel.theta0(fourier_params)

            a = np.concatenate((p * theta_n, p * theta_n, [p * theta_0]))
            b = np.concatenate((q * theta_n, q * theta_n, [q * theta_0]))
            c = r * np.ones(2 * len(n) + 1)
            d = np.concatenate((s - omega0 * n, s + omega0 * n, [s]))

            return (
                a, b, c, d
            )

    class FullProfileTerm(celerite.terms.Term):
        parameter_names = pnames

        def __init__(self, *args, **kwargs):
            super(FullProfileTerm, self).__init__(*args, **kwargs)
            self.timekernel = time_kernel(**extract_time_args(kwargs))

        def get_real_coefficients(self, params):
            time_params = params[time_mask]
            fourier_params = params[fourier_mask]
            test, _, _, _ = self.timekernel.get_complex_coefficients(time_params)
            if test.size > 0:
                # There are complex terms - we only will have complex terms
                return (np.empty(0), np.empty(0))
            else:
                # The kernel only has real terms, so we have some real terms.
                theta0 = phase_kernel.theta0(fourier_params)
                p, r = self.timekernel.get_real_coefficients(time_params)
                a = p * theta0
                c = r
                return (a, c)

        def get_complex_coefficients(self, params):
            time_params = params[time_mask]
            fourier_params = params[fourier_mask]
            p, q, r, s = self.timekernel.get_complex_coefficients(time_params)
            if p.size == 0:
                # Only real Terms...
                p, r = self.timekernel.get_real_coefficients(time_params)
                n, theta_n = phase_kernel(fourier_params)
                a = 2 * np.outer(p, theta_n).flatten()
                b = np.zeros_like(a)
                c = np.outer(r, np.ones_like(theta_n)).flatten()
                d = np.outer(omega0 * n, np.ones_like(p)).flatten()
                return (a, b, c, d)
            else:
                # Complex terms...
                # TODO: Handle the case where p,q,r,s have more than one value.
                n, theta_n = phase_kernel(fourier_params)
                theta_0 = phase_kernel.theta0(fourier_params)

                a = np.concatenate((p * theta_n, p * theta_n, [p * theta_0]))
                b = np.concatenate((q * theta_n, q * theta_n, [q * theta_0]))
                c = r * np.ones(2 * len(n) + 1)
                d = np.concatenate((s - omega0 * n, s + omega0 * n, [s]))
                return (a, b, c, d)

    if optimise_tau_real_only:
        return HalfProfileTerm
    else:
        return FullProfileTerm


import base64

from urllib.request import urlopen
from bs4 import BeautifulSoup
import pandas as pd
import datetime as datetime
from bokeh.plotting import figure, show, output_file, save
from bokeh.layouts import column
from bokeh.models import CDSView, GroupFilter, ColumnDataSource, RangeTool, HoverTool, TapTool, OpenURL
from google.cloud import storage
from google.cloud import bigquery
from google.cloud import pubsub_v1

def crawler(product,url):
  header = ["date","release_type","description","hyperlink"]
  releases = []
  html = urlopen(url)
  pot = BeautifulSoup(html,'lxml')
  bowl = pot.find('section',attrs={"class":"releases"})
  spoons = bowl.findAll(['h2','div','p','ul'])
  for spoon in spoons:
    if spoon.name == 'h2':
      keep_date = spoon.text #get('data-text')
      keep_link = url.strip()+'#'+spoon.get('id')
    if spoon.name == 'div':
      keep_type = spoon.get('class')[0]
      keep_disc = spoon.findNext('p').text
      releases.append([keep_date,keep_type,keep_disc,keep_link])
  df = pd.DataFrame(releases,columns=header)
  df.date = df.date.apply(lambda x: datetime.datetime.strptime(x,"%B %d, %Y"))
  df['product'] = product
  releasemap = {"release-fixed":"Fix","release-issue":"Issue","release-changed":"Change","release-feature":"Feature","release-announcement":"Announcement"}
  df['release_type'].replace(releasemap, inplace=True)
  return df

def write_bq(df):
  # parameters
  BQ_PROJECT = 'statmike-internal-site'
  BQ_DATASET = 'RELEASE_NOTES'
  BQ_TABLE = 'BQ_Release_Notes'
  BQ_REGION = 'us-central1'
  # client for bq
  bq_client = bigquery.Client(project=BQ_PROJECT)
  # look at datasets, create if needed
  datum = []
  for item in list(bq_client.list_datasets()): datum.append(item.dataset_id)
  if BQ_DATASET not in datum:
    dataset = bigquery.Dataset(bigquery.dataset.DatasetReference(BQ_PROJECT, BQ_DATASET))
    dataset.location = BQ_REGION
    dataset = bq_client.create_dataset(dataset)
  # create or replace table
  bq_job_config = bigquery.LoadJobConfig(schema=[],write_disposition="WRITE_TRUNCATE",)
  bq_job = bq_client.load_table_from_dataframe(df, "{}.{}.{}".format(BQ_PROJECT,BQ_DATASET,BQ_TABLE),job_config=bq_job_config)

def bq_plotter():
  pdict = {"bq":"BigQuery","bqml":"BigQuery ML","bqbi":"BigQuery BI Engine","bqdt":"BigQuery Data Transfer Service"}
  bq = crawler('bq','https://cloud.google.com/bigquery/docs/release-notes')
  bqml = crawler('bqml','https://cloud.google.com/bigquery-ml/docs/release-notes')
  bqbi = crawler('bqbi','https://cloud.google.com/bi-engine/docs/release-notes')
  bqdt = crawler('bqdt','https://cloud.google.com/bigquery-transfer/docs/release-notes')

  # combine the product data and sort by date descending
  df = pd.concat([bq,bqml,bqbi,bqdt], axis=0, ignore_index=True)
  df = df.sort_values(by=['date'], ascending=False, ignore_index=True)

  # update the BQ table with this info - replaces the previous run
  write_bq(df)

  # create the timeline plot
  # dictionaries for poduct labels and colors
  pdict = {"bq":"BigQuery","bqml":"BigQuery ML","bqbi":"BigQuery BI Engine","bqdt":"BigQuery Data Transfer Service"}
  colormap = {"bq":"#4285F4", "bqml":"#EA4335", "bqbi":"#FBBC04", "bqdt":"#34A853"}

  # define the source data for the plot
  source = ColumnDataSource(data=dict(date=df['date'], release=df['release_type'], tip=df['description'], url=df['hyperlink'],
                                      product=df['product'], productname=[pdict[x] for x in df['product']],
                                      colors=[colormap[x] for x in df['product']]))
  ycats = df.release_type.unique()

  # format date for tooltips
  source.add(df['date'].apply(lambda d: d.strftime('%m/%d/%Y')),'date_pretty')

  # create hover tools
  p_hover = HoverTool(mode='mouse', line_policy='nearest', names=['p_primary'],
                      tooltips=[("","@date_pretty @productname"),("","@tip")])

  # main plot = p
  p = figure(title="Big Query Release Notes",
            plot_height=300, plot_width=800, tools=["xpan","tap",p_hover], toolbar_location=None,
            x_axis_type="datetime", x_axis_location="above",
            background_fill_color="#F8F9FA", y_range=ycats, x_range=(df.date[100], df.date[0]))
  p.yaxis.axis_label = 'Release Type'

  #loop over products and display glyphs (circles), use CDSView to create view of product subset from source
  for prod in df['product'].unique():
    view = CDSView(source=source, filters=[GroupFilter(column_name='product', group=prod)])
    p.circle('date','release',source=source, view=view, line_color=None, size=10, fill_color='colors', legend_label=pdict[prod], name='p_primary')

  # configure taptool
  url="@url"
  taptool = p.select(type=TapTool)
  taptool.callback = OpenURL(url=url)

  # configure legend
  p.legend.location='top_left'
  p.legend.click_policy="hide"

  # selection tool below chart = select
  select = figure(title="Drag the middle and edges of the selection box to change the range above",
                  plot_height=130, plot_width=800, y_range=p.y_range,
                  x_axis_type="datetime", y_axis_type=None,
                  tools="", toolbar_location=None, background_fill_color="#F8F9FA")
  range_tool = RangeTool(x_range=p.x_range)
  range_tool.overlay.fill_color = "#5F6368"
  range_tool.overlay.fill_alpha = 0.2
  select.circle('date', 'release', source=source, line_color=None, fill_color='colors')
  select.ygrid.grid_line_color = None
  select.add_tools(range_tool)
  select.toolbar.active_multi = range_tool

  full = column(p,select)
  output_file("/tmp/bqplot.html")
  save(full)

  # Store the plot in GCS
  storage_client = storage.Client(project='statmike-internal-site')
  bucket = storage_client.get_bucket('statmike-internal-site')
  blob = bucket.blob('bq-timeline/bqplot.html')
  blob.upload_from_filename('/tmp/bqplot.html')
  blob.make_public()

  ## trigger pub/sub topic bq-forecast
  pub_client = pubsub_v1.PublisherClient()
  topic = pub_client.topic_path('statmike-internal-site','bq-forecast')
  mess = 'Proceed with Forecast'
  future = pub_client.publish(topic,mess.encode("utf-8"))

def bq_timeline(event, context):
    """Triggered from a message on a Cloud Pub/Sub topic.
    Args:
         event (dict): Event payload.
         context (google.cloud.functions.Context): Metadata for the event.
    """
    pubsub_message = base64.b64decode(event['data']).decode('utf-8')
    print(pubsub_message)

    bq_plotter()


import cv2
import math
import mediapipe as mp
from google.colab.patches import cv2_imshow
# Function to calculate the angle between two lines
def calculate_angle(line1, line2):
    dx1 = line1[2] - line1[0]
    dy1 = line1[3] - line1[1]
    dx2 = line2[2] - line2[0]
    dy2 = line2[3] - line2[1]
    angle = math.atan2(dx1 * dy2 - dy1 * dx2, dx1 * dx2 + dy1 * dy2)
    return math.degrees(angle)

# Load the image
image_path = '/content/img.jpeg'
image = cv2.imread(image_path)

# Face Landmark Detection using MediaPipe
mp_face_mesh = mp.solutions.face_mesh

with mp_face_mesh.FaceMesh(
        max_num_faces=1,
        refine_landmarks=True,
        min_detection_confidence=0.5,
        min_tracking_confidence=0.5
) as face_mesh:

    results = face_mesh.process(image)

    if results.multi_face_landmarks:
        # Get face landmarks
        face_landmarks = results.multi_face_landmarks[0]

        # Convert the normalized coordinates of landmarks to pixel coordinates
        landmark_points = [(int(landmark.x * image.shape[1]), int(landmark.y * image.shape[0])) for landmark in face_landmarks.landmark]

        # Initialize variables for right and left eye landmarks
        right_eye_coords = None
        left_eye_coords = None

        # Initialize variables for the topmost and lowest landmarks in the y-axis
        topmost_landmark = None
        lowest_landmark = None

        for i, point in enumerate(landmark_points):
            if i == 159 or i == 386:  # Right and left eye landmarks
                cv2.circle(image, point, 4, (0, 255, 0), -1)  # Draw green circles at eye landmarks
                if i == 159:
                    right_eye_coords = point
                else:
                    left_eye_coords = point
            else:
                cv2.circle(image, point, 4, (0, 0, 255), -1)  # Draw red circles at other landmarks

                # Find the topmost and lowest landmarks
                if topmost_landmark is None or point[1] < topmost_landmark[1]:
                    topmost_landmark = point
                if lowest_landmark is None or point[1] > lowest_landmark[1]:
                    lowest_landmark = point

        if right_eye_coords and left_eye_coords:
            # Calculate the midpoint of the line connecting the right and left eyes
            midpoint_x = (right_eye_coords[0] + left_eye_coords[0]) // 2
            midpoint_y = (right_eye_coords[1] + left_eye_coords[1]) // 2

            # Draw the midpoint in blue
            cv2.circle(image, (midpoint_x, midpoint_y), 4, (255, 0, 0), -1)  # Draw a blue circle at the midpoint

            # Draw a straight line between the green points (right and left eye landmarks)
            cv2.line(image, right_eye_coords, left_eye_coords, (0, 255, 0), 2)

            # Initialize a flag to check if the angle condition is met
            angle_condition_met = False

            # Try to meet the angle condition by changing the topmost and lowest landmarks
            while not angle_condition_met:
                # Calculate the angle between the line connecting the eyes and the line between topmost and lowest landmarks
                eye_line = [right_eye_coords[0], right_eye_coords[1], left_eye_coords[0], left_eye_coords[1]]
                top_low_line = [topmost_landmark[0], topmost_landmark[1], lowest_landmark[0], lowest_landmark[1]]
                angle = calculate_angle(eye_line, top_low_line)

                # Check if the angle is between 80 and 100 degrees
                if 70 <= angle <= 110:
                    angle_condition_met = True
                else:
                    # Change the topmost and lowest landmarks one at a time
                    if topmost_landmark[1] < lowest_landmark[1]:
                        topmost_landmark = (topmost_landmark[0], topmost_landmark[1] + 1)
                    else:
                        lowest_landmark = (lowest_landmark[0], lowest_landmark[1] + 1)

            # Draw a line between the eyes
            cv2.line(image, right_eye_coords, left_eye_coords, (255, 0, 0), 2)

            # Calculate the Euclidean distance between the topmost and lowest positions
            euclidean_distance = cv2.norm(topmost_landmark, lowest_landmark)

            # Print the Euclidean distance
            print(f"Euclidean Distance: {euclidean_distance}")

        # Color the topmost and lowest landmarks black
        cv2.circle(image, topmost_landmark, 4, (0, 0, 0), -1)
        cv2.circle(image, lowest_landmark, 4, (0, 0, 0), -1)

        # Draw a line between the topmost and lowest landmarks
        cv2.line(image, topmost_landmark, lowest_landmark, (0, 255, 0), 2)

        # Display the image with marked landmarks and lines
        cv2_imshow(image)
        cv2.waitKey(0)
        cv2.destroyAllWindows()

    else:
        print("No face detected in the image.")


# A python program for converting number to word
# Reference : https://stackoverflow.com/questions/19504350/how-to-convert-numbers-to-words-without-using-num2word-library

def number_to_word_convert(num):

    units = (
        "", "one ", "two ", "three ", "four ", "five ", "six ", "seven ", "eight ", "nine ", "ten ", "eleven ",
        "twelve ",
        "thirteen ", "fourteen ", "fifteen ", "sixteen ", "seventeen ", "eighteen ", "nineteen ")
    tens = ("", "", "twenty-", "thirty-", "forty-", "fifty-",
            "sixty-", "seventy-", "eighty-", "ninety-")
    # orders of magnitude (oom) added
    oom = ('thousand', 'million', 'billion', 'trillion', 'quadrillion', 'quintillion', 'sextillion', 'septillion',
           'octillion', 'nonillion', 'decillion', 'undecillion', 'duodecillion', 'tredecillion', 'quattuordecillion',
           'quindecillion', 'sexdecillion', 'septendecillion', 'octodecillion', 'novemdecillion', 'vigintillion')

    if num < 0:
        return "minus " + number_to_word_convert(-num)

    if num < 20:
        return units[num]

    if num < 100:
        return tens[num // 10] + units[num % 10]

    if num < 10 ** 3:
        return units[num // 10 ** 2] + "hundred and " + number_to_word_convert(num % 10 ** 2)

    for idx, name in enumerate(oom):
        scale = (idx + 1) * 3
        cap = scale + 3
        if num < 10 ** cap:
            return number_to_word_convert(num // 10 ** scale) + name + " " + number_to_word_convert(num % 10 ** scale)

    return "tends to infinity"

def number_to_word_convert2(number):
    from num2words import num2words
    return num2words(number)

number = int(input("Enter any integer number:"))

print(number_to_word_convert(number))

print(number_to_word_convert2(number))


class Solution(object):
    def canBeEqual(self, target, arr):
        """
        :type target: List[int]
        :type arr: List[int]
        :rtype: bool
        """
        # sort both arrays
        target.sort(reverse=True)
        arr.sort(reverse=True)
        if target==arr:
            return True
        else:
            return False

Easys/1460. Make Two Arrays Equal by Reversing Subarrays.py


import sys
import os

# add path that contains the dominoes package
mainPath = os.path.dirname(os.path.abspath(__file__)) + "/.."
sys.path.append(mainPath)

# standard imports
from copy import deepcopy
import argparse
from pathlib import Path
from tqdm import tqdm
import numpy as np
from scipy.signal import savgol_filter
from scipy.stats import ttest_rel
import matplotlib.pyplot as plt
import matplotlib as mpl
import torch
import torch.cuda as torchCuda

# dominoes package
from dominoes import fileManagement as fm
from dominoes import datasets
from dominoes import training
from dominoes import transformers
from dominoes.utils import loadSavedExperiment

device = 'cuda' if torchCuda.is_available() else 'cpu'

# general variables for experiment
POINTER_METHODS = ['PointerStandard', 'PointerDot', 'PointerDotLean', 'PointerDotNoLN', 'PointerAttention', 'PointerTransformer']

# path strings
netPath = fm.netPath()
resPath = fm.resPath()
prmsPath = fm.prmPath()
figsPath = fm.figsPath()

for path in (resPath, prmsPath, figsPath, netPath):
    if not(path.exists()):
        path.mkdir()

# method for returning the name of the saved network parameters (different save for each possible opponent)
def getFileName(extra=None):
    baseName = "ptrArchComp_TSP_RL"
    if hasattr(args, 'nobaseline') and not(args.nobaseline):
        baseName += "_withBaseline"
    if extra is not None:
        baseName = baseName + f"_{extra}"
    return baseName

def handleArguments():
    parser = argparse.ArgumentParser(description='Run pointer demonstration.')
    parser.add_argument('-nc','--num-cities', type=int, default=10, help='the number of cities')
    parser.add_argument('-bs','--batch-size',type=int, default=128, help='number of sequences per batch')
    parser.add_argument('-ne','--train-epochs',type=int, default=12000, help='the number of training epochs')
    parser.add_argument('-te','--test-epochs',type=int, default=200, help='the number of testing epochs')
    parser.add_argument('-nr','--num-runs',type=int, default=8, help='how many runs for each network to train')

    parser.add_argument('--nobaseline', default=False, action='store_true')
    parser.add_argument('--significance', default=0.05, type=float, help='significance of reward improvement for baseline updating')
    parser.add_argument('--baseline-batch-size', default=1024, type=int, help='the size of the baseline batch to use')

    parser.add_argument('--gamma', type=float, default=1.0, help='discounting factor')
    parser.add_argument('--temperature',type=float, default=5.0, help='temperature for training')

    parser.add_argument('--embedding_dim', type=int, default=128, help='the dimensions of the embedding')
    parser.add_argument('--heads', type=int, default=8, help='the number of heads in transformer layers')
    parser.add_argument('--encoding-layers', type=int, default=2, help='the number of stacked transformers in the encoder')
    parser.add_argument('--expansion', type=int, default=4, help='the expansion in the ff layer of the transformer')
    parser.add_argument('--justplot', default=False, action='store_true', help='if used, will only plot the saved results (results have to already have been run and saved)')
    parser.add_argument('--nosave', default=False, action='store_true')
    parser.add_argument('--printargs', default=False, action='store_true')

    args = parser.parse_args()

    assert args.gamma > 0 and args.gamma <= 1, "gamma must be greater than 0 or less than or equal to 1"
    assert args.significance > 0 and args.significance < 1, "significance must be greater than 0 or less than 1"

    return args

def resetBaselines(blnets, batchSize, num_cities, num_choices):
    # initialize baseline input (to prevent overfitting with training data)
    baselineinput, _, xy, baseline_dists = datasets.tsp_batch(batchSize, num_cities, return_target=False, return_full=True)
    baselineinput, xy, baseline_dists = baselineinput.to(device), xy.to(device), baseline_dists.to(device)
    baseline_start = torch.argmin(torch.sum(xy**2, dim=2), dim=1)
    baseline_init_input = torch.gather(baselineinput, 1, baseline_start.view(-1, 1, 1).expand(-1, -1, 2))

    # measure output of blnets for this data
    _, baseline_choices = map(list, zip(*[net((baselineinput, baseline_init_input), max_output=num_choices, init=baseline_start) for net in blnets]))
    baseline_full_choices = [torch.cat((baseline_start.view(-1, 1), choice), dim=1) for choice in baseline_choices]
    baseline_rewards, _ = map(list, zip(*[training.measureReward_tsp(baseline_dists, choice) for choice in baseline_full_choices]))
    baseline_rewards = [-reward for reward in baseline_rewards]

    return baselineinput, baseline_init_input, baseline_start, baseline_rewards, baseline_dists

def trainTestModel():
    # get values from the argument parser
    num_cities = args.num_cities
    num_in_cycle = num_cities
    num_choices = num_cities - 1 # initial position is defined

    # other batch parameters
    batchSize = args.batch_size
    baselineBatchSize = args.baseline_batch_size
    significance = args.significance
    do_baseline = not(args.nobaseline)

    # network parameters
    input_dim = 2
    embedding_dim = args.embedding_dim
    heads = args.heads
    encoding_layers = args.encoding_layers
    expansion = args.expansion
    temperature = args.temperature

    # train parameters
    trainEpochs = args.train_epochs
    testEpochs = args.test_epochs
    numRuns = args.num_runs

    numNets = len(POINTER_METHODS)

    # create gamma transform matrix
    gamma = args.gamma
    exponent = torch.arange(num_choices).view(-1,1) - torch.arange(num_choices).view(1,-1)
    gamma_transform = (gamma ** exponent * (exponent >= 0)).to(device)

    print(f"Doing training...")
    trainTourLength = torch.zeros((trainEpochs, numNets, numRuns))
    testTourLength = torch.zeros((testEpochs, numNets, numRuns))
    trainValidCycles = torch.zeros((trainEpochs, numNets, numRuns))
    testValidCycles = torch.zeros((testEpochs, numNets, numRuns))
    trainScoreByPosition = torch.full((trainEpochs, num_choices, numNets, numRuns), torch.nan) # keep track of confidence of model
    testScoreByPosition = torch.full((testEpochs, num_choices, numNets, numRuns), torch.nan)
    for run in range(numRuns):
        print(f"Training networks {run+1}/{numRuns}...")

        # create pointer networks with different pointer methods
        nets = [transformers.PointerNetwork(input_dim, embedding_dim, temperature=temperature, pointer_method=POINTER_METHOD,
                                            thompson=True, encoding_layers=encoding_layers, heads=heads, kqnorm=True,
                                            expansion=expansion, decoder_method='transformer', contextual_encoder='multicontext')
                for POINTER_METHOD in POINTER_METHODS]
        nets = [net.to(device) for net in nets]

        if do_baseline:
            # create baseline nets, initialized as copy of learning nets
            blnets = [deepcopy(net) for net in nets]
            for blnet in blnets:
                blnet.setTemperature(1.0)
                blnet.setThompson(True)

            baseline_data = resetBaselines(blnets, baselineBatchSize, num_cities, num_choices)
            baselineinput, baseline_init_input, baseline_start, baseline_rewards, baseline_dists = baseline_data

        # Create an optimizer, Adam with weight decay is pretty good
        optimizers = [torch.optim.Adam(net.parameters(), lr=1e-4, weight_decay=1e-6) for net in nets]

        for epoch in tqdm(range(trainEpochs)):
            # generate batch
            input, _, xy, dists = datasets.tsp_batch(batchSize, num_cities, return_target=False, return_full=True)
            input, xy, dists = input.to(device), xy.to(device), dists.to(device)

            # define initial position as closest to origin (could be arbitrary)
            start = torch.argmin(torch.sum(xy**2, dim=2), dim=1)
            init_input = torch.gather(input, 1, start.view(-1, 1, 1).expand(-1, -1, 2))

            # zero gradients, get output of network
            for opt in optimizers: opt.zero_grad()
            log_scores, choices = map(list, zip(*[net((input, init_input), max_output=num_choices, init=start) for net in nets]))
            full_choices = [torch.cat((start.view(-1, 1), choice), dim=1) for choice in choices] # add initial position

            # log-probability for each chosen dominoe
            logprob_policy = [torch.gather(score, 2, choice.unsqueeze(2)).squeeze(2) for score, choice in zip(log_scores, choices)]

            # measure rewards
            rewards, new_city = map(list, zip(*[training.measureReward_tsp(dists, choice)
                                                for choice in full_choices])) # distance penalized negatively (hence the -)
            rewards = [-reward for reward in rewards]
            chosen_rewards = [reward[:, 1:].contiguous() for reward in rewards]
            G = [torch.matmul(reward, gamma_transform) for reward in chosen_rewards]
            for i, (l, nc) in enumerate(zip(rewards, new_city)):
                assert not torch.any(torch.isnan(l)), f"model type {POINTER_METHODS[i]} diverged :("
                assert torch.all(nc==1), f"model type {POINTER_METHODS[i]} didn't do a permutation"

            if do_baseline:
                # - do "baseline rollout" with baseline networks -
                with torch.no_grad():
                    _, bl_choices = map(list, zip(*[net((input, init_input), max_output=num_choices, init=start) for net in blnets]))
                    bl_full_choices = [torch.cat((start.view(-1, 1), choice), dim=1) for choice in bl_choices]

                    bl_rewards, _ = map(list, zip(*[training.measureReward_tsp(dists, choice) for choice in bl_full_choices]))
                    bl_rewards = [-reward for reward in bl_rewards]
                    bl_chosen_rewards = [reward[:, 1:].contiguous() for reward in bl_rewards]
                    bl_G = [torch.matmul(reward, gamma_transform) for reward in bl_chosen_rewards]

                    adjusted_G = [g-blg for g, blg in zip(G, bl_G)] # baseline corrected G
            else:
                # for a consistent namespace
                adjusted_G = [g for g in G]

            # measure J using adjusted G ()
            J = [-torch.sum(logpol * g) for logpol, g in zip(logprob_policy, adjusted_G)] # flip sign for gradient ascent
            for j in J: j.backward()

            # update networks
            for opt in optimizers: opt.step()

            # measure position dependent error
            with torch.no_grad():
                for i in range(numNets):
                    trainTourLength[epoch, i, run] = torch.mean(torch.sum(rewards[i], dim=1))
                    trainValidCycles[epoch, i, run] = torch.mean(1.0*torch.all(new_city[0]==1, dim=1))

                    # Measure the models confidence -- ignoring the effect of temperature
                    pretemp_score = torch.softmax(log_scores[i] * nets[i].temperature, dim=2)
                    pretemp_policy = torch.gather(pretemp_score, 2, choices[i].unsqueeze(2)).squeeze(2)
                    trainScoreByPosition[epoch, :, i, run] = torch.mean(pretemp_policy, dim=0).detach()

            # check if we should update baseline networks
            if do_baseline:
                with torch.no_grad():
                    # first measure policy on baseline data (in evaluation mode)
                    for net in nets:
                        net.setTemperature(1.0)
                        net.setThompson(False)

                    _, choices = map(list, zip(*[net((baselineinput, baseline_init_input), max_output=num_choices, init=baseline_start) for net in nets]))
                    full_choices = [torch.cat((baseline_start.view(-1, 1), choice), dim=1) for choice in choices]

                    rewards, _ = map(list, zip(*[training.measureReward_tsp(baseline_dists, choice) for choice in full_choices]))
                    rewards = [-reward for reward in rewards]
                    _, p = map(list, zip(*[ttest_rel(r.view(-1).cpu().numpy(), blr.view(-1).cpu().numpy(), alternative='greater')
                                            for r, blr in zip(rewards, baseline_rewards)]))
                    do_update = [pv<significance for pv in p]

                    # for any networks with significantly different values, update them
                    for ii, update in enumerate(do_update):
                        if update:
                            blnets[ii] = deepcopy(nets[ii])
                            blnets[ii].setTemperature(1.0)
                            blnets[ii].setThompson(False)

                    # regenerate baseline data and get baseline network policy
                    if any(do_update):
                        baseline_data = resetBaselines(blnets, baselineBatchSize, num_cities, num_choices)
                        baselineinput, baseline_init_input, baseline_start, baseline_rewards, baseline_dists = baseline_data

                    # return nets to training state
                    for net in nets:
                        net.setTemperature(temperature)
                        net.setThompson(True)

        with torch.no_grad():
            for net in nets:
                net.setTemperature(1.0)
                net.setThompson(False)

            print("testing...")
            for epoch in tqdm(range(testEpochs)):
                # generate batch
                input, _, xy, dists = datasets.tsp_batch(batchSize, num_cities, return_target=False, return_full=True)
                input, xy, dists = input.to(device), xy.to(device), dists.to(device)

                # define initial position as closest to origin (could be arbitrary)
                start = torch.argmin(torch.sum(xy**2, dim=2), dim=1)
                init_input = torch.gather(input, 1, start.view(-1, 1, 1).expand(-1, -1, 2))

                # get output of network
                log_scores, choices = map(list, zip(*[net((input, init_input), max_output=num_choices, init=start) for net in nets]))
                full_choices = [torch.cat((start.view(-1, 1), choice), dim=1) for choice in choices] # add initial position

                # log-probability for each chosen dominoe
                logprob_policy = [torch.gather(score, 2, choice.unsqueeze(2)).squeeze(2) for score, choice in zip(log_scores, choices)]

                # measure rewards
                rewards, new_city = map(list, zip(*[training.measureReward_tsp(dists, choice)
                                                    for choice in full_choices])) # distance penalized negatively (hence the -)
                rewards = [-reward for reward in rewards]
                for i, (l, nc) in enumerate(zip(rewards, new_city)):
                    assert not torch.any(torch.isnan(l)), f"model type {POINTER_METHODS[i]} diverged :("
                    assert torch.all(nc==1), f"model type {POINTER_METHODS[i]} didn't do a permutation"

                # measure position dependent error
                with torch.no_grad():
                    for i in range(numNets):
                        testTourLength[epoch, i, run] = torch.mean(torch.sum(rewards[i], dim=1))
                        testValidCycles[epoch, i, run] = torch.mean(1.0*torch.all(new_city[0]==1, dim=1))

                        # Measure the models confidence -- ignoring the effect of temperature
                        pretemp_score = torch.softmax(log_scores[i] * nets[i].temperature, dim=2)
                        pretemp_policy = torch.gather(pretemp_score, 2, choices[i].unsqueeze(2)).squeeze(2)
                        testScoreByPosition[epoch, :, i, run] = torch.mean(pretemp_policy, dim=0).detach()

    results = {
        'trainTourLength': trainTourLength,
        'testTourLength': trainTourLength,
        'trainValidCycles': trainValidCycles,
        'testValidCycles': testValidCycles,
        'trainScoreByPosition': trainScoreByPosition,
        'testScoreByPosition': testScoreByPosition
    }

    return results, nets

def plotResults(results, args):
    numRuns = args.num_runs
    cmap = mpl.colormaps['tab10']

    # make plot of loss trajectory
    fig, ax = plt.subplots(1,2,figsize=(6,4), width_ratios=[2.6,1],layout='constrained')
    for idx, name in enumerate(POINTER_METHODS):
        cdata = torch.nanmean(-results['trainTourLength'][:,idx], dim=1)
        idx_nan = torch.isnan(cdata)
        cdata.masked_fill_(idx_nan, 0)
        cdata = savgol_filter(cdata, 10, 1)
        cdata[idx_nan] = torch.nan
        ax[0].plot(range(args.train_epochs), cdata, color=cmap(idx), lw=1.2, label=name)
    ax[0].set_xlabel('Training Epoch')
    ax[0].set_ylabel(f'Tour Length (N={numRuns})')
    ax[0].set_title(f'Training Performance')
    ax[0].legend(loc='best')

    xOffset = [-0.2, 0.2]
    get_x = lambda idx: [xOffset[0]+idx, xOffset[1]+idx]
    for idx, name in enumerate(POINTER_METHODS):
        mnTestReward = torch.nanmean(-results['testTourLength'][:,idx], dim=0)
        ax[1].plot(get_x(idx), [mnTestReward.mean(), mnTestReward.mean()], color=cmap(idx), lw=4, label=name)
        for mtr in mnTestReward:
            ax[1].plot(get_x(idx), [mtr, mtr], color=cmap(idx), lw=1.5)
        ax[1].plot([idx,idx], [mnTestReward.min(), mnTestReward.max()], color=cmap(idx), lw=1.5)
    ax[1].set_xticks(range(len(POINTER_METHODS)))
    ax[1].set_xticklabels([pmethod[7:] for pmethod in POINTER_METHODS], rotation=45, ha='right', fontsize=8)
    ax[1].set_ylabel(f'Tour Length (N={numRuns})')
    ax[1].set_title('Testing')
    ax[1].set_xlim(-1, len(POINTER_METHODS))

    if not(args.nosave):
        plt.savefig(str(figsPath/getFileName('tourLength')))

    plt.show()

    # make plot of number of valid cycles
    fig, ax = plt.subplots(1,2,figsize=(6,4), width_ratios=[2.6,1],layout='constrained')
    for idx, name in enumerate(POINTER_METHODS):
        cdata = torch.nanmean(results['trainValidCycles'][:,idx], dim=1)
        idx_nan = torch.isnan(cdata)
        cdata.masked_fill_(idx_nan, 0)
        cdata = savgol_filter(cdata, 10, 1)
        cdata[idx_nan] = torch.nan
        ax[0].plot(range(args.train_epochs), cdata, color=cmap(idx), lw=1.2, label=name)
    ax[0].set_xlabel('Training Epoch')
    ax[0].set_ylabel(f'Fraction Valid (N={numRuns})')
    ax[0].set_title(f'Training Complete Cycles')
    ax[0].legend(loc='best')

    xOffset = [-0.2, 0.2]
    get_x = lambda idx: [xOffset[0]+idx, xOffset[1]+idx]
    for idx, name in enumerate(POINTER_METHODS):
        mnTestReward = torch.nanmean(results['testValidCycles'][:,idx], dim=0)
        ax[1].plot(get_x(idx), [mnTestReward.mean(), mnTestReward.mean()], color=cmap(idx), lw=4, label=name)
        for mtr in mnTestReward:
            ax[1].plot(get_x(idx), [mtr, mtr], color=cmap(idx), lw=1.5)
        ax[1].plot([idx,idx], [mnTestReward.min(), mnTestReward.max()], color=cmap(idx), lw=1.5)
    ax[1].set_xticks(range(len(POINTER_METHODS)))
    ax[1].set_xticklabels([pmethod[7:] for pmethod in POINTER_METHODS], rotation=45, ha='right', fontsize=8)
    ax[1].set_ylabel(f'Fraction Valid (N={numRuns})')
    ax[1].set_title('Testing')
    ax[1].set_xlim(-1, len(POINTER_METHODS))

    if not(args.nosave):
        plt.savefig(str(figsPath/getFileName('tourComplete')))

    plt.show()

    # now plot confidence across positions
    numPos = results['testScoreByPosition'].size(1)
    fig, ax = plt.subplots(1, 2, figsize=(6,4), width_ratios=[2.6,1], layout='constrained')
    for idx, name in enumerate(POINTER_METHODS):
        ax[0].plot(range(numPos), torch.mean(results['testScoreByPosition'][:,:,idx], dim=(0,2)), color=cmap(idx), lw=1, marker='o', label=name)
    ax[0].set_xlabel('Output Position')
    ax[0].set_ylabel('Mean Score')
    ax[0].set_title('Confidence')
    ax[0].legend(loc='best', fontsize=8)
    ax[0].set_ylim(0.93, 1)

    xOffset = [-0.2, 0.2]
    get_x = lambda idx: [xOffset[0]+idx, xOffset[1]+idx]
    for idx, name in enumerate(POINTER_METHODS):
        mnScoreByPosition = torch.mean(results['testScoreByPosition'][:,:,idx], dim=(0,1))
        ax[1].plot(get_x(idx), [mnScoreByPosition.mean(), mnScoreByPosition.mean()], color=cmap(idx), lw=4, label=name)
        for msbp in mnScoreByPosition:
            ax[1].plot(get_x(idx), [msbp, msbp], color=cmap(idx), lw=1.5)
        ax[1].plot([idx,idx], [mnScoreByPosition.min(), mnScoreByPosition.max()], color=cmap(idx), lw=1.5)
    ax[1].set_xticks(range(len(POINTER_METHODS)))
    ax[1].set_xticklabels([pmethod[7:] for pmethod in POINTER_METHODS], rotation=45, ha='right', fontsize=8)
    ax[1].set_title('Average')
    ax[1].set_xlim(-1, len(POINTER_METHODS))
    ax[1].set_ylim(0.93, 1)

    if not(args.nosave):
        plt.savefig(str(figsPath/getFileName(extra='confidence')))

    plt.show()

if __name__=='__main__':
    args = handleArguments()
    show_results = True

    if args.printargs:
        _, args = loadSavedExperiment(prmsPath, resPath, getFileName(), args=args)
        for key, val in vars(args).items():
            print(f"{key}={val}")
        show_results = False

    elif not(args.justplot):
        # train and test pointerNetwork
        results, nets = trainTestModel()

        # save results if requested
        if not(args.nosave):
            # Save agent parameters
            for net, method in zip(nets, POINTER_METHODS):
                save_name = f"{method}.pt"
                torch.save(net, netPath / getFileName(extra=save_name))
            # Save agent parameters
            np.save(prmsPath / getFileName(), vars(args))
            np.save(resPath / getFileName(), results)

    else:
        results, args = loadSavedExperiment(prmsPath, resPath, getFileName(), args=args)

    if show_results:
        plotResults(results, args)


import os
import cv2
import time
from utils import iou
from scipy import spatial
from darkflow.net.build import TFNet

options = {'model': 'cfg/tiny-yolo-voc-3c.cfg',
           'load': 3750,
           'threshold': 0.1,
           'gpu': 0.7}

tfnet = TFNet(options)

avg_time = 0
pred_bb = []  # predicted bounding box
pred_cls = []  # predicted class
pred_conf = []  # predicted class confidence

directory = 'dataset/Testing/Images/'

for file_name in os.listdir(directory):
    tic = time.time()
    image = cv2.imread(directory + file_name)
    output = tfnet.return_predict(image)

    rbc = 0
    wbc = 0
    platelets = 0

    cell = []
    cls = []
    conf = []

    record = []
    tl_ = []
    br_ = []
    iou_ = []
    iou_value = 0

    for prediction in output:
        label = prediction['label']
        confidence = prediction['confidence']

        tl = (prediction['topleft']['x'], prediction['topleft']['y'])
        br = (prediction['bottomright']['x'], prediction['bottomright']['y'])

        if label == 'RBC' and confidence < .5:
            continue
        if label == 'WBC' and confidence < .25:
            continue
        if label == 'Platelets' and confidence < .25:
            continue

        # clearing up spurious platelets
        if label == 'Platelets':
            if record:
                tree = spatial.cKDTree(record)
                index = tree.query(tl)[1]
                iou_value = iou(tl + br, tl_[index] + br_[index])
                iou_.append(iou_value)

            if iou_value > 0.1:
                continue

            record.append(tl)
            tl_.append(tl)
            br_.append(br)

        # image = cv2.rectangle(image, tl, br,color, 2)
        center_x = int((tl[0] + br[0]) / 2)
        center_y = int((tl[1] + br[1]) / 2)
        center = (center_x, center_y)

        if label == 'RBC':
            color = (255, 0, 0)
            rbc = rbc + 1
        if label == 'WBC':
            color = (0, 255, 0)
            wbc = wbc + 1
        if label == 'Platelets':
            color = (0, 0, 255)
            platelets = platelets + 1

        radius = int((br[0] - tl[0]) / 2)
        image = cv2.circle(image, center, radius, color, 2)
        font = cv2.FONT_HERSHEY_COMPLEX
        image = cv2.putText(image, label, (center_x - 15, center_y + 5), font, .5, color, 1)
        cell.append([tl[0], tl[1], br[0], br[1]])

        if label == 'RBC':
            cls.append(0)
        if label == 'WBC':
            cls.append(1)
        if label == 'Platelets':
            cls.append(2)

        conf.append(confidence)

    pred_bb.append(cell)
    pred_cls.append(cls)
    pred_conf.append(conf)
    cv2.imwrite('output/' + file_name, image)
    toc = time.time()
    avg_time = avg_time + (toc - tic) * 1000

avg_time = avg_time / 60

print('Mean time: {0:.5}'.format(avg_time), 'ms')
print('All Done!')


import dotenv,os

def get_value(conf,key):
    # type: (object, object) -> object
    "Return the value in conf for a given key"
    value = None

    try:
        dotenv.load_dotenv(conf)
        value = os.environ[key]
    except Exception as e:
        print("Exception in get_value")
        print('file: ',conf)
        print('key: ',key)

        return value


import csv

with open("data.csv","w+") as csvfile:
    writer = csv.writer(csvfile)
    writer.writerow(["Title","Description"])
    writer.writerow(["row 1","Descript"])
    writer.writerow(["row 2","Description"])
    writer.writerow(["row 3","Descript"])

# "a" Append to a file--- add to the very end
# "w" writes to the file, w+ overwrites to the file
# "r" read only
# "r+" read & write
# "rb" read in binary form

with open("data.csv","r") as csvfile:
    reader = csv.reader(csvfile)
    for row in reader:
        print(row)

with open("data.csv","a") as csvfile:
    writer = csv.writer(csvfile)
    writer.writerow(["hi", "my name is","johnny doe"])
    writer.writerow(["hi", "my name is","johnny doe"])
    writer.writerow(["hi", "my name is","johnny doe"])
    writer.writerow(["hi", "my name is","johnny doe"])

with open("data.csv","r") as csvfile:
    reader = csv.DictReader(csvfile)
    for row in csv.DictReader(csvfile):
        print(row)

CSV-Files-with-Python - Read-Write-Append/CSV- Read-Write-Append.py