# -*- coding: utf-8 -*-

import sys
import codecs
from win32com.client import Dispatch
from collections import defaultdict
from mapping import fields, tables
from utils import dump_to_json_file, data_dir, log_dir

sub = '\n       ' #'|'
sep = '\t|'
sep_short = '|'
tag_start = '\n'
tag_end = '' #'%'
record_start = '='*100 + '\n'
record_end = '\x1d\n\n'

MAX_RECS = 500000
BATCH_SIZE = 1024
table = "T.CATADO"


def get_field(node):
    if not node:
        return ""
    if node.NodesCount == 0:
        # If a field has one occurrence...
        return node.Value
    # If field has two or more occurrences
    # Loop through each occurrence
    s = ""
    node_type = node.Nodes(1)[0].NodeType if isinstance(node.Nodes(1), tuple) else node.Nodes(1).NodeType
    if node_type == 3:
        # First Child node is Occurrence, so display occurrences
        for i in range(1, node.NodesCount+1):
            n = node.Nodes(i)
            if isinstance(n, tuple):
                n = n[0]
            if n != None and n.NodesCount == 0 and  n.Value != None:
                s += sub + (n.Label if n.Label else ' ') + sep_short + n.Value.strip()
            else:
                for j in range(1, n.NodesCount+1):
                    nj = n.Nodes(j)
                    if isinstance(nj, tuple):
                        nj = nj[0]
                    if nj != None and nj.NodesCount == 0 and nj.Value != None:
                        s += sub + nj.Label + sep_short + nj.Value.strip()

    else:
        # Iterate through Subfields
        for i in range(1, node.NodesCount+1):
            n = node.Nodes(i)
            if isinstance(n, tuple):
                n = n[0]
            if n != None and n.Value != None:
                s += sub + n.Label + sep_short + n.Value.strip()
    return s

def update_progress(progress):
    sys.stdout.write('\r[{0}{1}] {2}% '.format('#'*(int(progress)),
                     ' '*(100-int(progress)),
                     round(progress, 2)))
    sys.stdout.flush()

def dump_to_text_file(d, filename):
    print(">>> Writing to text file...")
    total = float(len(d.items()))
    f = open(data_dir + filename, 'w')
    out = ''
    counter = 0
    for r in sorted(d.items()):
        out = record_start + 'RECNO' + sep + str(r[0]) + tag_end
        for tag in sorted(r[1].items()):
            if tag[1] != '' and tag[0] != 'RECNO':
                out += tag_start + tag[0] + sep + tag[1] + tag_end
        out += record_end
        f.write(out.encode('utf-8'))
        counter = counter + 1
        update_progress(counter*100/total)
    f.close()

def format_record(d):
    out = record_start + 'RECNO' + sep + str(d['RECNO']) if 'RECNO' in d else '' + tag_end
    if d.items():
        # out += record_start
        for tag in sorted(d.items()):
            if tag[0] != 'RECNO':
                out += tag_start + tag[0] + sep + tag[1] + tag_end
        out += record_end
    return out

def main():
    item_type = sys.argv[1]
    MAX_RECS = 2500000 #int(sys.argv[2])
    tables = {
        'vendors': ['T.LIBVENDORS'],
        'orders':  ['T.ACQPOS', 'T.ACQITEMS'], #'T.ACQITEMS2'], #['T.ACQFUNDS', 'T.ACQINVOICE', 'T.ACQISELRCI', 'T.ACQISELRCV', 'T.ORDERSTAT', 'T.ORDERTYPES'],
        'patrons': ['T.APPUSERS'], # 'T.IAEAUSERS'],
        'serials': ['T.SERHOLD', 'T.SERIALS', ], # 'T.SERROUTE', 'T.SFREQS', ], # ['T.SERSELEOYR', 'T.SERSELHOLD', 'T.SERSELVUP'],
        'bibliographic': ['T.CATADO'],  # Items
        'checkouts': ['T.CATCIRC'],
        'holds': ['T.RESPICK'],
        }
    with codecs.open(log_dir + 'log_export.txt', 'w', encoding='utf8') as logfile:
        logfile.close()
    tables_to_download = []
    if item_type in tables:
        tables_to_download = tables[item_type]
    else:
        tables_to_download = [item_type]

    for table in tables_to_download:
        print(">>> Extracting from %s..." % table)
        try:
            i = 1
            batch = 10000
            last_record = 110000
            last_fetched = 0
            recs = {}
            errors = []
            get_me_out_of_here = False
            offset = 0
            while last_fetched < last_record:
                print("\n>>> Connecting (%d)..." % i)
                conn = Dispatch("STARADO.Connection")
                conn.Login("libstar", "11002", "jaime", "dedalus")

                res = conn.OpenRecordset(table, "R>%d AND R<=%d" % (last_fetched, min(last_fetched+batch, last_record)), " ".join(fields[table]))

                rs = res[0]
                if rs:
                    rec_no = rs.RecordCount
                    print(">>> Downloading %s records. [ %d to %d ]" % (min(rec_no, MAX_RECS), last_fetched+1, min(last_fetched+batch, last_record)))
                    total = float(min(rec_no, MAX_RECS))
                    if rs.RecordCount > 0:
                        f = open(data_dir+table+'_'+str(i)+'.txt', 'w')
                        rs.BatchSize = BATCH_SIZE  # Set number of records to be retrieved in cached batches.
                        rs.MoveFirst()  # Move focus to first record of retrieved set.
                        field_list = rs.FieldList.split(' ')
                        # Loop through each record until end of set is reached.
                        counter = 1
                        while not rs.EOF:
                            d = defaultdict(lambda : None)
                            for field in field_list:
                                value = None
                                try:
                                    value = get_field(rs.Record(field))
                                    if value != None and value != '':
                                        d[field] = value
                                except:
                                    with codecs.open(log_dir + 'log_export.txt', 'a', encoding='utf8') as logfile:
                                        logfile.write(("==================\n%d\n%s:\t%s\n\n" % (counter+offset, field, value if value else 'No value set yet')).replace('\n', '\r\n'))
                                        logfile.close()
                            # try:
                            f.write(format_record(d).encode('utf-8'))
                            recs[counter+offset] = d
                            # print counter*i
                            rs.MoveNext()  # Go to the next record in the set
                            update_progress(counter*100/total)
                            counter = counter + 1
                            # if counter >= MAX_RECS:
                            #     break
                        offset = offset + rs.RecordCount
                        rs.MoveLast()
                        f.close()
                    else:
                        get_me_out_of_here = True
                    last_fetched = last_fetched + rs.RecordCount #batch #int(get_field(rs.Record("RECNO")))
                    i = i + 1
                    # else:
                    print("\n>>> Closing connection. This may take a while...")
                    rs.CloseRecordset()
                    conn.Logout()
                    conn.CloseConnection()
                    rs = None
                    if get_me_out_of_here:
                        break
                else:
                    conn.Logout()
                    conn.CloseConnection()
                    break
            # print("\n>>> Saving json...")
            # dump_to_json_file(recs, table+'.json')
            # dump_to_text_file(recs, table+'.txt')
        except Exception as e:
            print e
            print("\n>>> There was an ERROR: saving everything...")
            # rs.Cancel()
            f.close()
            # rs.CloseRecordset()
            conn.Logout()
            conn.CloseConnection()
            rs = None

        if len(recs):
            dump_to_json_file(recs, table+'.json')
            # dump_to_json_file(errors, table+'-errors.json')
            # print("\n>>> There were %d errors") % len(errors)
            print(">>> %d records fetched") % len(recs)
            # sys.exit(0)


if __name__ == '__main__':
    main()

#
# ['T.ACQFUNDS',  'T.ACQFUNDSED', 'T.ACQFUNDSEO', 'T.ACQFUNDSOL', 'T.ACQINVOICE',
# 'T.ACQISELRCI', 'T.ACQISELRCV', 'T.ACQITEMS',   'T.ACQITEMSO1', 'T.ACQITEMSO2',
# 'T.ACQITEMSW',  'T.ACQPOS',     'T.ACQPOS1',    'T.ACQPOSBAK',  'T.ACQPOSED',
# 'T.ACQTEST',    'T.APPUPDATE',  'T.APPUSERS',   'T.APPUSERSN',  'T.APPUSERSSN',
# 'T.APPUSERSSV', 'T.AV',         'T.AVUSE',      'T.BOILERPLAT', 'T.CAT',        'T.CATADDITEM',
# 'T.CATBAK',     'T.CATCIRC',    'T.CATIN',      'T.CATINS',     'T.CATLINK',    'T.CATMASTER',
# 'T.CATOUT',     'T.CATOUTS',    'T.CATOVERDUE', 'T.CATPARTS',   'T.CATPRO',
# 'T.CATPROADIT', 'T.CATPROANAL', 'T.CATPROBKS',  'T.CATPROSER',  'T.CATPROSP',
# 'T.CATPROTOC',  'T.CATRENEW',   'T.CATRENEWS',  'T.CATRES',     'T.CATSELREQ',
# 'T.CATSELREQA', 'T.CATSELREQC', 'T.CATSELREQW', 'T.CATSP',      'T.CATTITLES',
# 'T.CATTOC',     'T.CATWEB',     'T.CATWEB4',    'T.CATWEBDES',  'T.CATWEBSEL',
# 'T.CATWEBSRV',  'T.CLAIMINT',   'T.CLSAMPTHES', 'T.COLLECTION', 'T.COUNTERS',
# 'T.CURRENCY',   'T.DIVISION',   'T.EMAIL',      'T.EMAILDB',    'T.GLOBAL',     'T.GLOBAL1',
# 'T.GLOBAL2',    'T.GLOBAL3',    'T.GLOBAL4',    'T.HNPU',       'T.IAEACSBC',   'T.IAEAUSERS',
# 'T.INISCAT',    'T.INISCATV17', 'T.INISCATWEB', 'T.INISTHES',   'T.INVENTORY',
# 'T.ISSCODE',    'T.ISSDATES',   'T.ISSDATESEL', 'T.ISSN',       'T.LDOCKWS',    'T.LDOCKWSN',
# 'T.LIBSCFORMS', 'T.LIBSGLOBAL', 'T.LIBSSERVER', 'T.LIBTRACK',   'T.LIBTYPES',
# 'T.LIBVENDORS', 'T.LNUMTYPES',  'T.LREFREQS',   'T.LREFREQSC',  'T.LSELRESCAN',
# 'T.LSERVICES',  'T.LSERVICESP', 'T.LSERVICESR', 'T.LSERVICESW', 'T.LSERVNUMS',
# 'T.MARCTEMP',   'T.ORDERSTAT',  'T.ORDERTYPES', 'T.PICKCTRY',   'T.PICKCTRYED',
# 'T.PICKLANG',   'T.PICKLANGED', 'T.PICKSTEP',   'T.PROJECTS',   'T.PWSELECTS',
# 'T.RAINBOW',    'T.REGLIBUSE',  'T.REQTYPE',    'T.RESERVE',    'T.RESERVEC',   'T.RESPICK',
# 'T.SERHOLD',    'T.SERHOLD2',   'T.SERIALS',    'T.SERIALS2',   'T.SERIALSBK',
# 'T.SERIALSC1',  'T.SERIALSC2',  'T.SERIALSC3',  'T.SERIALSDM',  'T.SERIALSID',
# 'T.SERIALSID2', 'T.SERIALSIN',  'T.SERIALSNE',  'T.SERIALSNE2', 'T.SERIALSNT',
# 'T.SERIALSNT2', 'T.SERIALSUNX', 'T.SERROUTE',   'T.SERROUTE2',  'T.SERSELEOYR',
# 'T.SERSELHOLD', 'T.SERSELVUP',  'T.SFREQS',     'T.SP',         'T.STAFPUB',    'T.STATISTIC',
# 'T.STATS',      'T.SUBINFO',    'T.SUBSNO',     'T.TOPIC',      'T.TUNCAT',     'T.TUNCATTITL',
# 'T.TUNCATWEB4', 'T.TUNTITLES',  'T.VALDB',      'T.VALIDATION', 'T.WAITCODES',
# 'T.WEBDES',     'T.WEBPROFILE', 'T.WEBSAMPTHS', 'T.WEBSERVER',  'T.WEBTYPES',
# 'T.WHENX',      'T.WHENXSUPER',]
