import bs4      # BeautifulSoup HTML parser
import urllib2  # open URLs
import json     # store data in a portable way

def parseData(htmlData, timeTable):
    """
    The purpose of this function is to extract the interessting data from the 
    preloaded HTML page. This is done by using BeautifulSoup
    """
    soup = bs4.BeautifulSoup(htmlData)

    title = soup.find('title') # extract title and date information
    title = title.text

    start= title.find('am')    # find begin of date string
    if start != -1:
        date = title[start+2:].strip().split()
    else:
        start= title.find('-')
        date = title[start+2:].strip().split()

    # extract all tables with a summary indictaing the people of the debate
    table = soup.findAll('table', {'class': 'tabelle tabelleHistorie', \
                                   'summary' : 'Rednerinnen und Redner der Debatte'})

    for tab in table: # iterate over all tables extract the required information
        topic = tab.find_previous_sibling("h3",{'style': 'padding-top: 0.5em;'}).text
        fields = tab.findAll('td')
        for i in range(len(fields)/9):
            try:
                #print fields[i*9+4].text.strip()
                tmp = fields[i*9+6].text.strip().replace(':',' ').split()
                if fields[i*9+2].text.strip() not in timeTable.keys():
                    timeTable[fields[i*9+2].text.strip()] = []
                timeTable[fields[i*9+2].text.strip()].append([int(tmp[0])*60 + int(tmp[1]),topic,fields[i*9+4].text.strip()])
            except:
                continue

            #print fields[i*9+2].text.strip(), fields[i*9+6].text.strip()
    return date

def parseFullSet(s, timeTable):
    """
    iterate over a full legislature period, this hast to be specified using
    roman numbers.
    """

    # allow up to 300 debate days, this is VERY unlikely to be exceeded
    for i in range(300):
        try:
            page = 'http://www.parlament.gv.at/PAKT/VHG/{0}/NRSITZ/NRSITZ_{1:0>5}/index.shtml'.format(s,i)
            print page
            webpage = urllib2.urlopen(page)
            table = {}
            date = parseData(webpage,table)
            print len(date), date
            if len(date) != 3:
                date = timeTable[-1][2:5]
                print "neues d=", date
            timeTable.append([s,i,date[0],date[1],date[2],table])

        # in case the URL doesn't exist we ignore it
        except urllib2.URLError:
            continue
        

        

if __name__ == "__main__":
    timeTable = []
    
    parseFullSet('XX',timeTable)
    parseFullSet('XXI',timeTable)
    parseFullSet('XXII',timeTable)
    parseFullSet('XXIII',timeTable)
    parseFullSet('XXIV',timeTable)
    parseFullSet('XXV',timeTable)

    f = open("speakerTimeTable.json", "w")
    json.dump(timeTable, f)
    f.close()
    #for i in timeTable:
    #    for j in i[5]:
    #        print i[0], i[1], i[2], i[3], i[4], j, i[5][j]
    
    #print timeTable
