text_html_output_1.png
from bs4 import BeautifulSoup

from pathlib import Path

import pandas as pd

import plotly.express as px

PUB = Path('../input/30dmlleaderboards/public_lb.html')
PRIV = Path('../input/30dmlleaderboards/private_lb.html')
CSV_PUB = Path('../input/30dmlleaderboards/30-days-of-ml-publicleaderboard/30-days-of-ml-publicleaderboard.csv')

def _strip_all_spaces(series):
    return series.replace('\\s+', ' ', regex=True).str.strip()
def _extract_kernel(element):
    try:
        anchor = element.find('a')
        title = anchor.get('title')
        href = anchor.get('href')
    except AttributeError:
        title, href = ('', '')
    return (title, href)
def _load_pub_dataframe(pathname=PUB):
    """
    Scrape the key data items from the HTML version of the public leaderboard.
    Return a pandas DataFrame
    """
    with pathname.open() as f:
        soup = BeautifulSoup(f, 'lxml-xml')
    recs = soup.find_all('tr', class_=['competition-leaderboard__row', 'competition-leaderboard__row competition-leaderboard__row--user-scored'])
    rows = []
    for rec in recs:
        record = dict()
        keys = ['Rank', 'Team Name', 'Kernel', 'Score', 'Number of Entries']
        for key in keys:
            if key != 'Kernel':
                record[key] = rec.find('td', {'data-th': key}).text
            else:
                element = rec.find('td', {'data-th': key})
                record['KernelTitle'], record['KernelHref'] = _extract_kernel(element)
        record['Last Entry'] = rec.find_all('span')[-1]['title']
        rows.append(record)
    df = pd.DataFrame(rows)
    df['Last Entry'] = pd.to_datetime(df['Last Entry'].str.split().str[0:6].str.join(' '))
    df['Team Name'] = _strip_all_spaces(df['Team Name'])
    df = df[df['Team Name'].notna()]
    df.columns = ['Rank', 'TeamName', 'KernelTitle', 'KernelHref', 'Score', 'Entries', 'Latest']
    df = df.drop(['Score', 'Latest'], axis='columns')
    return df
def _load_priv_dataframe(pathname=PRIV):
    """
    Scrape the key data items from the HTML version of the private leaderboard.
    Return a pandas DataFrame
    """
    with pathname.open() as f:
        soup = BeautifulSoup(f, 'lxml-xml')
    recs = soup.find_all('tr', class_=['competition-leaderboard__row', 'competition-leaderboard__row competition-leaderboard__row--user-scored'])
    rows = []
    for rec in recs:
        record = dict()
        keys = ['Rank', 'Team Name', 'Score']
        for key in keys:
            record[key] = rec.find('td', {'data-th': key}).text
        change_span = rec.find('td', {'data-th': 'Change'}).find('span')
        if change_span.find('span', class_='position-change__none'):
            change = (0, 0)
        elif change_span.find('span', class_='position-change__risen'):
            change = (1, int(change_span.find('span', class_='position-change__risen').text))
        else:
            change = (-1, int(change_span.find('span', class_='position-change__fallen').text))
        record['ChangeDirection'], record['ChangeNo'] = change
        rows.append(record)
    df = pd.DataFrame(rows)
    df['Team Name'] = _strip_all_spaces(df['Team Name'])
    df.columns = ['PrivRank', 'TeamName', 'PrivScore', 'ChangeDirection', 'ChangeNo']
    return df
def _load_csv_dataframe(pathname=CSV_PUB):
    """
    Read the CSV version of the public leaderboard as downloaded from Kaggle.
    (This will be merged with the scraped version).
    Return a pandas DataFrame
    """
    df = pd.read_csv(pathname)
    df.TeamName = df.TeamName.replace('\\s+', ' ', regex=True).str.strip()
    return df
def _load_dataframe(pathname):
    if pathname == PUB:
        df = _load_pub_dataframe()
    elif pathname == CSV_PUB:
        df = _load_csv_dataframe()
    return df
def _load_and_merge_public_lb():
    """
    Load and clean the two versions (HTML and CSV) of the public leaderboard.
    Then merge the two on TeamName.
    Return a merged pandas DataFrame.
    """
    df = _load_dataframe(PUB)
    df_pub = _load_dataframe(CSV_PUB)
    df = df[df['TeamName'].isin(df_pub['TeamName'])]
    df_pub = df_pub[df_pub['TeamName'].isin(df['TeamName'])]
    final = df.merge(df_pub, on='TeamName', how='left')
    final.columns = ['PubRank', 'TeamName', 'KernelTitle', 'KernelHref', 'Entries', 'TeamId', 'SubmissionDate', 'PubScore']
    return final
def load_data():
    """
    Load all data (two versions of public and scraped version of private leaderboard).
    Merge into one DataFrame and set dtypes correctly.
    Return a pandas DataFrame
    """
    pub_df = _load_and_merge_public_lb()
    priv_df = _load_priv_dataframe()
    df = pub_df.merge(priv_df, on='TeamName', how='left')
    type_dict = {'PubRank': 'int32', 'Entries': 'int32', 'SubmissionDate': 'datetime64', 'PubScore': 'float64', 'PrivRank': 'int32', 'PrivScore': 'float64', 'ChangeDirection': 'category', 'ChangeNo': 'int32'}
    for key, value in type_dict.items():
        df[key] = df[key].astype(value)
    new_order = ['TeamId', 'TeamName', 'PubRank', 'PubScore', 'PrivRank', 'PrivScore', 'ChangeDirection', 'ChangeNo', 'SubmissionDate', 'Entries', 'KernelTitle', 'KernelHref']
    df = df[new_order]
    return df

DF = load_data()
DF.sample(5)

RANK = 500
SOURCE = DF[DF['PrivRank'] <= RANK].copy()

x_data = SOURCE["Entries"]
y_data = SOURCE["PrivRank"]
color_data = SOURCE["ChangeDirection"]
size_data = SOURCE["Entries"]

fig = px.scatter(SOURCE, x=x_data, y=y_data, color=color_data, 
                 size=size_data, opacity=0.75, hover_data=['TeamName'])
fig.show()

x_data = SOURCE["SubmissionDate"]
y_data = SOURCE["Entries"]
color_data = SOURCE["PrivRank"]
size_data = SOURCE["Entries"]

fig = px.scatter(SOURCE, x=x_data, y=y_data, color=color_data, size=size_data, opacity=0.75, hover_data=['TeamName'])
fig.show()

x_data = SOURCE["PrivRank"]
y_data = SOURCE["PrivScore"]
color_data = SOURCE["ChangeDirection"]
size_data = SOURCE["Entries"]

fig = px.scatter(SOURCE, x=x_data, y=y_data, color=color_data, size=size_data, opacity=0.75, hover_data=['TeamName'])
fig.show()

x_data = SOURCE['PubRank']
y_data = SOURCE['PubScore']
color_data = SOURCE['ChangeDirection']
size_data = SOURCE['Entries']
fig = px.scatter(SOURCE, x=x_data, y=y_data, color=color_data, size=size_data, opacity=0.75, hover_data=['TeamName'])
fig.show()