<repo_name>cssko/UC-CourseCatalog<file_sep>/README.md
# UC-CourseCatalog
*A central course catalog for the University of Cincinnati*

All things considered the University of Cincinnati has a fairly navigable [website](www.uc.edu). However, one common complaint by staff and students both has been a lack of a central course catalog. Each college within the university typically has this information somewhere, but it isn't always up–to–date. There's also the [Course Planning Guide](http://webapps.uc.edu/registrar/courseplanningguide/SemesterCoursePlanningGuide.aspx), but this has a clunky interface.

My goal here is to provide a central and easily searchable course catalog. Users will search using a course number (e.g. MATH1061) and be provided with a course title and description.
<file_sep>/courseScraper.py
import random
import time
import sqlite3

from selenium.common.exceptions import NoSuchElementException
from selenium import webdriver
from selenium.webdriver.support.ui import Select
import selenium.webdriver.support.ui as UI


def insert_class_info(course_id, discipline, course_title, course_desc, credit_hours, fall_availability,
                      spring_availability, summer_availability, alt_yr, grad):
    """ Inserts or updates course information in sqlite3 database.

    :param course_id: Course ID
    :param discipline: Discipline the course falls under
    :param course_title: Title of course
    :param course_desc: Description of course
    :param credit_hours: Number of credit hours for course
    :param fall_availability: Fall semester availability of course
    :param spring_availability: Spring semester availability of course
    :param summer_availability: Summer semester availability of course
    :param alt_yr: Alternating Year availability for course
    :return:
    """
    cursor.execute("INSERT OR REPLACE INTO coursecatalog_course VALUES (?,?,?,?,?,?,?,?,?,?)", (
        course_id, discipline, course_title, course_desc, credit_hours, fall_availability, spring_availability,
        summer_availability, alt_yr, grad))
    conn.commit()
    return


def get_course_information(course_id, link, links, skipped_rows, discipline):
    """ Gets course information for current course.

    :param course_id: Course ID of the current course
    :param link: Link for the current Course ID
    :param links: List of links
    :param skipped_rows: Number of skipped rows, based on pagination.
    :param discipline: Discipline
    :return:
    """
    popup = driver.window_handles[1]
    driver.switch_to.window(popup)

    course_title = unicode((driver.find_element_by_id("CourseTitle")).text).strip()
    course_desc = unicode(driver.find_element_by_id("CourseDes").text).strip(course_id).strip()

    time.sleep(2 + (random.random() * 3))
    driver.close()
    driver.switch_to.window(driver.window_handles[0])
    # Make a list of the column items for the current row being worked on.
    info = driver.find_elements_by_css_selector('#ctl00_StaticPlaceHolder_CourseGridView '
                                                'tr:nth-child(' + str((links.index(link) + skipped_rows)) + ') td+ td')
    credit_hours = info[1].text
    fall_availability = info[2].text.replace(" ", "N").replace("ConsultNDepartment", "Consult Department")
    spring_availability = info[3].text.replace(" ", "N").replace("ConsultNDepartment", "Consult Department")
    summer_availability = info[4].text.replace(" ", "N").replace("ConsultNDepartment", "Consult Department")
    alt_yr = info[5].text.replace(" ", "N").replace("ConsultNDepartment", "Consult Department")
    insert_class_info(course_id, discipline, course_title, course_desc, credit_hours, fall_availability,
                      spring_availability, summer_availability, alt_yr, graduate_level)
    return


def navigate_courses(discipline):
    """ Moves through the courses in a discipline.
    Changes pages if needed.
    :param discipline: Current discipline
    :return:
    """
    try:  # Checks for links at the top of the class table, which will only exist if there are more than one page.
        driver.find_element_by_css_selector('tr+ tr td:nth-child(2) a')
        pagination = True
    except NoSuchElementException:
        # To my knowledge selenium lacks the ability to determine if an element exists without looking for it
        # and throwing an exception if it does not.
        pagination = False

    pages = 1
    cont = True

    while cont:
        links = driver.find_elements_by_partial_link_text(str(discipline))
        # If the discipline is PD then webdriver is going to want to click on 'Save PDF' and throw itself off.
        # This way we save itself the trouble.
        if discipline == "PD":
            for i in links:
                if i.text == "Save to PDF":
                    del links[links.index(i)]
                    break

        # If there is more than one page we want to start a row lower than otherwise.
        if pagination:
            skipped_rows = 3
        else:
            skipped_rows = 2

        for link in links:
            if links.__sizeof__() < 1:
                cont = False
                break

            time.sleep(3 + (random.random() * 2))
            course_id = str(link.text)
            link.click()
            get_course_information(course_id, link, links, skipped_rows, discipline)

        if pagination:
            try:  # Attempts to click on link that matches pages
                pages += 1
                if pages == 11:
                    driver.execute_script("javascript:__doPostBack('ctl00$StaticPlaceHolder$CourseGridView'"
                                          ",'Page$11')")  # Page 11 is a special snowflake and has the text '...'
                else:
                    next_page = driver.find_element_by_link_text(str(pages))
                    next_page.click()
            except NoSuchElementException:
                # Returns to the first page and breaks out of loop. Going back to the first page is necessary with how
                # the university's site works. If you go from one discipline to another, and they both have multiple
                # pages, you can end up on a later page
                driver.execute_script("javascript:__doPostBack('ctl00$StaticPlaceHolder$CourseGridView','Page$1')")
                cont = False
        elif not pagination:
            driver.execute_script("javascript:__doPostBack('ctl00$StaticPlaceHolder$CourseGridView','Page$1')")
            cont = False
        time.sleep(2 + (random.random() * 2))
    # time.sleep(5 + (random.random() * 5))
    return


def get_disciplines():
    """ Creates list of disciplines from drop down menu.
    :return: list of disciplines
    """
    disciplines = []
    disc_values = UI.Select(
        driver.find_element_by_xpath('//*[@id="ctl00_StaticPlaceHolder_DisciplineDDL"]'))
    for option in disc_values.options:
        disciplines.append(option.get_attribute("value"))
    del disciplines[0]  # The first option in the drop down list is blank and needs to be discarded
    # Prompt asking if you've already successfully completed some scraping and you wish to start at a
    # different discipline.
    start = raw_input('Input starting discipline: ').upper()
    for i in range(0, disciplines.index(start)):
        del disciplines[0]
    return disciplines


def select_grad_level():
    """ Prompts for desired graduate level.
    :return: String containing U or G. """
    grad = raw_input('Select graduate level.\nEnter U for undergraduate or G for graduate: ').upper()
    cont = True
    while cont:
        if grad == 'U' or grad == 'G':
            cont = False
        else:
            grad = raw_input('Incorrect input. Please enter either U or G: ').upper()
    Select(driver.find_element_by_id("ctl00_StaticPlaceHolder_GradTypeDDL")).select_by_value(grad)
    return grad


def main():
    global graduate_level
    grad = select_grad_level()
    # Here we take the users input in the form of a single letter string and expand it out into the full word
    # For readabilities sake.
    if grad == 'U':
        graduate_level = 'undergraduate'
    elif grad == 'G':
        graduate_level = 'graduate'

    disciplines = get_disciplines()
    for discipline in disciplines:
        Select(driver.find_element_by_id("ctl00_StaticPlaceHolder_DisciplineDDL")).select_by_value(discipline)
        time.sleep(2 + (random.random() * 3))
        print('Working on ' + discipline + " now!")
        navigate_courses(discipline)
    driver.close()


if __name__ == '__main__':
    url = "http://webapps.uc.edu/registrar/courseplanningguide/SemesterCoursePlanningGuide.aspx"
    while 1:
        browser = raw_input('Enter desired browser, either phantomjs or chrome: ').lower()
        if browser == 'chrome':
            path_to_chromedriver = "./chromedriver"
            driver = webdriver.Chrome(executable_path=path_to_chromedriver)
            driver.get(url)
            break
        elif browser == 'phantomjs':
            dcap = dict()
            dcap["phantomjs.page.settings.userAgent"] = ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53 "
                                                    "(KHTML, like Gecko) Chrome/15.0.87")
            path_to_phantomjs = './phantomjs'
            driver = webdriver.PhantomJS(executable_path=path_to_phantomjs, desired_capabilities=dcap)
            driver.get(url)
            break
        # else:
        #     raw_input('Please enter either "chrome" or "phantomjs": ')

    conn = sqlite3.connect("coursedata.db")
    cursor = conn.cursor()
    main()
