# fifteenwordsnippets.py

# A script that searches a HathiTrust corpus of 6,942 volumes (1700-1923), plus Hoyt & Richard's
# corpus of 808 vols (1923-1950), for words related to money. It takes seven words on either
# side of those words to create a snippet.

# In cases where the possibly-monetary word is ambiguous, e.g. "pounds," it runs the central
# seven words of the snippet through a regularized logistic model (created by model_contexts)
# in order to make a prediction about the likelihood that this word refers to money. The
# model I used is based on 700 manually-tagged snippets; it's about about 87% accurate,
# five-fold crossvalidated.

import modelingcounter
import os, sys
import SonicScrewdriver as utils
import csv
import pickle
from bagofwords import WordVector, StandardizingVector
from sklearn.linear_model import LogisticRegression

# We start with a number of functions borrowed from other scripts; these were used to
# generate the logistic model, so we use them also here to clean and normalize snippets.

punctuple = ('.', ',', '?', '!', ';', '"', '“', '”', ':', '--', '—', ')', '(', "'", "`", "[", "]", "{", "}")

def all_nonalphanumeric(astring):
    nonalphanum = True
    for character in astring:
        if character.isalpha() or character.isdigit():
            nonalphanum = False
            break
    return nonalphanum

def strip_punctuation(astring):
    global punctuple
    keepclipping = True
    suffix = ""
    while keepclipping == True and len(astring) > 1:
        keepclipping = False
        if astring.endswith(punctuple):
            suffix = astring[-1:] + suffix
            astring = astring[:-1]
            keepclipping = True
    keepclipping = True
    prefix = ""
    while keepclipping == True and len(astring) > 1:
        keepclipping = False
        if astring.startswith(punctuple):
            prefix = prefix + astring[:1]
            astring = astring[1:]
            keepclipping = True
    return(prefix, astring, suffix)

def as_wordlist(line):
    ''' Converts a line into a list of words, splitting
    tokens brutally and unreflectively at punctuation.
    One of the effects will be to split possessives into noun
    and s. But this might not be a bad thing for current
    purposes.
    '''

    line = line.replace('”', ' ')
    line = line.replace(':', ' ')
    line = line.replace(';', ' ')
    line = line.replace('—', ' ')
    line = line.replace('--', ' ')
    line = line.replace('.', ' ')
    line = line.replace(',', ' ')
    line = line.replace('-', ' ')
    line = line.replace('—', ' ')
    line = line.replace("'", ' ')
    line = line.replace('"', ' ')

    # That's not the most efficient way to do this computationally,
    # but it prevents me from having to look up the .translate
    # method.

    words = line.split(' ')

    wordlist = list()

    for word in words:
        word = word.lower()
        prefix, word, suffix = strip_punctuation(word)
        # In case we missed anything.

        if len(word) > 0 and not all_nonalphanumeric(word):
            wordlist.append(word)

    return wordlist

def is_money(wordlist, WINDOWRADIUS, model, features, standardizer):
    # We're getting a wordlist generated by WINDOWRADIUS, but
    # we only want the central seven words for our model.

    startindex = WINDOWRADIUS - 3
    endindex = WINDOWRADIUS + 4

    modelsegment = ' '.join(wordlist[startindex : endindex])

    # You'd think we could just take a list of words, but in
    # generating the model we ran strings through a particular
    # tokenizing process, and we should replicate that here.

    normalizedlist = as_wordlist(modelsegment)
    vector = WordVector(normalizedlist)
    vector.selectfeatures(features)
    # This turns a sparse dictionary into an array with zeroes
    # for missing features.

    vector.normalizefrequencies()
    # raw counts are divided by total counts.

    vector.standardizefrequencies(standardizer)
    # features are now centered on the means, and divided by
    # standard deviations, calculated on the training set

    classlabel = model.predict(vector.features)[0]

    if classlabel == 1:
        return True
    elif classlabel == 0:
        return False
    else:
        print("ANOMALY!")
        print(classlabel)
        return False

        # Cause that's how I do error handling.

# Main script.

# Let's load the model.

modelfolder = "/Volumes/TARDIS/work/moneycontext/"
modelpath = modelfolder + "logisticmodel.p"
with open(modelpath, mode = 'rb') as f:
    logisticmodel = pickle.load(f)

standardizerpath = modelfolder + 'standardizer.p'
with open(standardizerpath, mode = 'rb') as f:
    standardizer = pickle.load(f)

featurepath = modelfolder + 'featurelist.p'
with open(featurepath, mode = 'rb') as f:
    features = pickle.load(f)

# Now load HathiTrust metadata.

rows, columns, table = utils.readtsv('/Volumes/TARDIS/work/metadata/MergedMonographs.tsv')

ambiguouswords = {'crown', 'crowns', 'guinea', 'guineas', 'nickel', 'sovereign', 'sovereigns', 'pound', 'pounds', 'quid'}

moneywords = {'dollar', 'dollars', 'dime', 'dimes', 'nickel', 'nickels', 'pound', 'pounds', 'shilling', 'shillings', 'sovereign', 'sovereigns','cent', 'cents', 'centime', 'centimes', 'crown', 'crowns', 'halfcrown', 'half-crown','penny', 'pennies', 'pence', 'farthing', 'farthings', 'franc', 'francs', 'guilder', 'guilders', 'florin', 'florins', 'guinea', 'guineas', "ha'penny", 'tuppence', 'twopence', 'sixpence', '|arabicprice|', '|price|', 'quid'}

# Words I explicitly decided not to include: 'quarter', 'quarters', 'mark', 'marks.' Monetary uses
# seemed rare enough relative to others that they'd be more likely to introduce noise than to help.
# |arabicprice| is a code the tokenizer in modelingcounter produces whenever it encounters
# a number connected to £, $, ¢, s, or d. In the output we convert that to |price|, for no very
# good reason.

wealthwords = {'fortune', 'fortunes', 'wealth', 'rich', 'riches', 'money', 'moneys', 'fund', 'funds', 'sum', 'sums', 'price', 'prices', 'priced'}

# This is by no means an exhaustive list. Owe, loan, borrowed, etc.
# If we really want to get at the full range of words potentially
# associated with money, topic modeling would be an appropriate lever.
# We can perhaps enumerate currency terms intuitively, but not these.

alltargetwords = moneywords

sourcedir = "/Volumes/TARDIS/work/moneytexts/"
filelist = os.listdir(sourcedir)
filelist = [x for x in filelist if x.endswith(".txt")]
contexts = []

WINDOWRADIUS = 7

ctr = 0

for filename in filelist:

    htid = utils.pairtreelabel(filename.replace('.fic.txt', ''))

    if htid not in rows:
        print(htid)
        continue
    else:
        date = utils.simple_date(htid, table)

    filepath = os.path.join(sourcedir, filename)
    with open(filepath, encoding = 'utf-8') as f:
        filelines = f.readlines()
    pagelist = [filelines]

    # The wordcounter module expects a list of pages, each of which is a list of lines.
    # Ebooks have no pages -- at least as I currently receive them -- so we treat it
    # all as one giant page.

    tokenstream = modelingcounter.makestream(pagelist)

    newcontexts = modelingcounter.extract_snippets(tokenstream,  WINDOWRADIUS, alltargetwords)

    approvedcontexts = []

    for snippet, snippettomodel in newcontexts:

        keyword = snippettomodel[WINDOWRADIUS]
        keyword = keyword.lower()
        prefix, keyword, suffix = strip_punctuation(keyword)

        if keyword in wealthwords:
            category = 'wealth'
        elif keyword in ambiguouswords:
            currency = is_money(snippettomodel, WINDOWRADIUS, logisticmodel, features, standardizer)
            if currency:
                category = 'money'
            else:
                category = "notmoney"
        elif keyword in moneywords:
            category = 'money'
        else:
            print('ANOMALY: ' + keyword)
            # Cause that's how I do error handling.
            category = 'null'

        if category == 'money':
            approvedcontexts.append((htid, date, snippet, keyword, category))

    print(ctr)
    ctr += 1

    outfile = "/Volumes/TARDIS/work/moneycontext/twentyfivesnippets.tsv"
    with open(outfile, mode='a', encoding='utf-8') as f:
        for context in approvedcontexts:
            htid, date, alist, keyword, category = context
            snippet = " ".join(alist)
            snippet = snippet.replace('\t', '')
            # Because we don't want stray tabs in our tab-separated values.

            f.write(htid + '\t' + str(date) + '\t' + keyword + '\t' + category + '\t' + snippet + '\n')


sourcedir = "/Volumes/TARDIS/work/US_NOVELS_1923-1950/"

filelist = os.listdir(sourcedir)
fileset = set([x for x in filelist if x.endswith(".txt")])
filelist = list(fileset)

metafile = os.path.join(sourcedir, "US_NOVELS_1923-1950_META.txt")

datedict = dict()
dateset = set()

with open(metafile, newline='', encoding = 'utf-8') as f:
    reader = csv.reader(f)
    for fields in reader:
        idcode = fields[0]
        date = int(fields[8])
        datedict[idcode] = date
        dateset.add(date)

for filename in filelist:

    htid = utils.pairtreelabel(filename.replace('.txt', ''))

    if htid not in datedict:
        print(htid)
        continue
    else:
        date = datedict[htid]

    filepath = os.path.join(sourcedir, filename)
    with open(filepath, encoding = 'utf-8') as f:
        filelines = f.readlines()
    pagelist = [filelines]

    # The wordcounter module expects a list of pages, each of which is a list of lines.
    # Ebooks have no pages -- at least as I currently receive them -- so we treat it
    # all as one giant page.

    tokenstream = modelingcounter.makestream(pagelist)

    newcontexts = modelingcounter.extract_snippets(tokenstream, WINDOWRADIUS, alltargetwords)
    approvedcontexts = []

    for snippet, snippettomodel in newcontexts:

        keyword = snippettomodel[WINDOWRADIUS]
        keyword = keyword.lower()
        prefix, keyword, suffix = strip_punctuation(keyword)

        if keyword in wealthwords:
            category = 'wealth'
        elif keyword in ambiguouswords:
            currency = is_money(snippettomodel, WINDOWRADIUS, logisticmodel, features, standardizer)
            if currency:
                category = 'money'
            else:
                category = "notmoney"
        elif keyword in moneywords:
            category = 'money'
        else:
            print('ANOMALY: ' + keyword)
            # Cause that's how I do error handling.
            category = 'null'

        if category == 'money':
            approvedcontexts.append((htid, date, snippet, keyword, category))

    outfile = "/Volumes/TARDIS/work/moneycontext/twentyfivesnippets.tsv"
    with open(outfile, mode='a', encoding='utf-8') as f:
        for context in approvedcontexts:
            htid, date, alist, keyword, category = context
            snippet = " ".join(alist)
            snippet = snippet.replace('\t', '')
            # Because we don't want stray tabs in our tab-separated values.

            f.write(htid + '\t' + str(date) + '\t' + keyword + '\t' + category + '\t' + snippet + '\n')








