<file_sep># -*- coding: utf-8 -*-
"""
Created on Mon Jan 21 21:50:58 2019

@author: <NAME>

NFL Big Data Bowl

"""

import os
import re
import pandas as pd
import numpy as np
from scipy import stats
from math import sqrt

pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)

os.chdir('C:/Users/Brad/Desktop/NFL Data/Big-Data-Bowl-master/Big-Data-Bowl-master/Data')

df = pd.read_csv('plays.csv')

# Exclude penalty and special teams plays
df = df[(df['isPenalty'] == False) & (df['SpecialTeamsPlayType'].isnull())]

# Exclude interceptions, sack plays, and QB runs
# (include complete pass plays, incomplete pass plays, and run plays)
df = df[df['PassResult'].isin(['C', 'I', np.nan])]

# sometimes two-point conversion attempts have a NA for SpecialTeamsPlayType
# and a False for isSTPlay
# Only way to catch them is to exclude down == 0 plays
df[(df['down'] == 0) & (df['SpecialTeamsPlayType'].isnull()) & (df['isSTPlay'] == False)]['playDescription']
df = df[df['down'] != 0]

# Exclude fumble plays (not case sensitive)
df = df[~df['playDescription'].str.contains("fumble", case = False)]



# Determine receiver and ball carrier

def get_rusher(description):
    """
    Purpose: collect rusher name on a given play
             
    Input: play description from plays.csv
    Output: rusher name
    """
    try:
        return list(re.findall(r'\w+[\.]+\w+', description))[0]
    except IndexError:
        # Found a description with no player names
        return np.nan
    
def get_receiver(description):
    """
    Purpose: collect receiver name on a given play
             
    Input: play description from plays.csv
    Output: receiver name
    """
    try:
        return list(re.findall(r'\w+[\.]+\w+', description))[1]
    except IndexError:
        # Found a description with no player names
        return np.nan

# Create rushers and receivers columns
rushers = df['playDescription'].apply(get_rusher).where(df['PassResult'].isnull())

receivers = df['playDescription'].apply(get_receiver).where(df['PassResult'].isin(['C', 'I']))

# Quick look
rushers.value_counts()[0:25]
receivers.value_counts()[0:25]

# Add columns
df.insert(loc = 25, column = 'Receiver', value = receivers)
df.insert(loc = 26, column = 'Rusher', value = rushers)

# Create a yards gain vs. needed column to help determine run play success
gained_needed = (round(df['PlayResult']/df['yardsToGo'], 2)).where(df['PassResult'].isnull())

# Add column
df.insert(loc = 27, column = 'GainNeed', value = gained_needed)

# Run play success decision rules require knowing which team is home
# and which is visitor
# Collect home and visitor abbreviations in a dictionary
#  gameId: (home, visitor)
games = pd.read_csv('games.csv')

home_visitor = {}

for i in range(games.shape[0]):
    gameId = games.iloc[i]['gameId']
    home = games.iloc[i]['homeTeamAbbr']
    visitor = games.iloc[i]['visitorTeamAbbr']
    home_visitor[gameId] = (home, visitor)
    
# Combine running back play success rules into a function
def rb_success(row, games):
    """
    Purpose: calcualte if a run play was a success based on decision rules
    
    Decision rules:
        - A play counts as a "Success" if it gains 40% of yards to go on first
          down, 60% of yards to go on second down, and 100% of yards to go on
          third or fourth down.
          
        - If the team is behind by more than a touchdown in the fourth quarter,
          the benchmarks switch to 50%/65%/100%.
          
        - If the team is ahead by any amount in the fourth quarter, the
          benchmarks switch to 30%/50%/100%.
    
    inputs:
        - An entire row from plays.csv with GainNeed column
        - a dict of gameId: (home, visitor)
        * Assumes down can take a value of 1,2,3 or 4 (not 0)
        
    output:
        - True if criteria met for success on run play
        - False if criteria not met
        - nan if not a running play
        
    Note: This function will not win any beauty contests
    """
    gameId = row['gameId']
    quarter = row['quarter']
    down = row['down']
    poss_team = row['possessionTeam']
    gain_need = row['GainNeed']
    
    if pd.isna(gain_need):
        # Not a run play
        return np.nan
    
    #print()
    #print(gain_need)
    
    if poss_team == games[gameId][0]:
        diff = row['HomeScoreBeforePlay'] - row['VisitorScoreBeforePlay']
    elif poss_team == games[gameId][1]:
        diff = row['VisitorScoreBeforePlay'] - row['HomeScoreBeforePlay']
    else:
        print('Error: posession team not found')
        
    ahead_by = 0
    down_by = 0
    
    if diff < 0:
        down_by = abs(diff)
    elif diff > 0:
        ahead_by = diff
    else:
        ahead_by = 0
    
    
    if (quarter == 4) and (down_by >= 6):
        if down == 1:
            return gain_need >= 0.5
        elif down == 2:
            return gain_need >= 0.65
        else:
            return gain_need >= 1.0

    elif (quarter == 4) and (ahead_by > 0):
        if down == 1:
            return gain_need >= 0.3
        elif down == 2:
            return gain_need >= 0.5
        else:
            return gain_need >= 1.0

        
    elif down == 1:
        return gain_need >= 0.4
    elif down == 2:
        return gain_need >= 0.6
    else:
        return gain_need >= 1.0
    
    #return np.nan

df = df.reset_index(drop = True)

run_success = pd.Series(np.nan, index = range(df.shape[0]))

for i in range(df.shape[0]):
    run_success[i] = rb_success(df.iloc[i], home_visitor)

# Add column
df.insert(loc = 28, column = 'RunSuccess', value = run_success)

# check to see if any touchdown runs are labeled as failures
df[df['playDescription'].str.contains('touchdown', case = False)][['quarter', 'down', 'GainNeed', 'RunSuccess', 'playDescription']]

success_rate = df.groupby('Rusher')['RunSuccess'].apply(lambda x: round((x*1).sum()/x.shape[0], 2))

rushes = df.groupby('Rusher')['RunSuccess'].count()

successful = df.groupby('Rusher')['RunSuccess'].apply(lambda x: round((x*1).sum()))

rb_success_rate = pd.DataFrame({'Rushes': rushes, 'Successful': successful, 'SuccessRate':success_rate})
rb_success_rate.sort_values('SuccessRate', inplace = True, ascending = False)
rb_success_rate.to_csv('success_rate.csv')


def ci_lower_bounds(successful_runs, n, confidence):
    """
    successful_runs: total runs multiplied by success rate
    n: total runs
    confidence: desired confidence level (e.g., 0.95)
    """
    if n == 0:
        return 0

    z = stats.norm.ppf(1-(1-0.95)/2)
    
    p_hat = 1.0*successful_runs/n
    
    return (p_hat + z*z/(2*n) - z*sqrt((p_hat*(1 - p_hat) + z*z/(4*n))/n))/(1 + z*z/n)



# sorted by confidence in success rate

rb_success_rate['Conf_Adj_Rank'] = rb_success_rate.apply(lambda x: round(ci_lower_bounds(x['Successful'], x['Rushes'], 0.95), 4), axis = 1)

rb_success_rate.sort_values('Conf_Adj_Rank', inplace = True, ascending = False)
rb_success_rate.to_csv('success_rate_adj.csv')



# Gather run plays
#Group subset of run plays by game ID, collect associated plays
df_rush = df[df['RunSuccess'].isin([True, False])]
rushes_by_gameId = df_rush[['playId', 'gameId', 'RunSuccess', 'Rusher', 'PlayResult']].groupby('gameId')


# Build a rush plays dictionary to associate games with a list of plays
# format:
#       key: gameId
#       value: [(plays.csv_index, playId, RunSuccess, Rusher, ...]
rush_plays = {}

for name, group in rushes_by_gameId:

    rush_plays[name] = list(zip(list(group.index),
              list(group['playId']),
              list(group['RunSuccess']),
              list(group['PlayResult']),
              list(group['Rusher'])))
    
def tot_dist_time(play):
    """
    Total distance player traveled on the play
    Total time to go that distance
    
    Use for computing efficiency and average speed columns
    
    Input: pandas groupby object; grouped by playId
    Output: distance, time
        - yards gained will come from the PlayResult column in the game_rush
          dataframe
    """
    
    stop_index = play['event'].last_valid_index()
    stop_event = play['event'][stop_index]
    
    if stop_event == 'qb_kneel':
        return np.nan
    
    dist = 0
    time = 0
    moving = False

    
    for i in range(play.shape[0]):
        #try:
            #print(play['event'][i])
        #except KeyError:
            #print("failed on )
        
        # We do not know which man 'man_in_motion' refers to
        # Use 'ball_snap' as start of distance
        if play['event'].iloc[i] == 'ball_snap' or play['event'].iloc[i] == 'snap_direct':
            moving = True
            
        # final non NA value marks the end of the run
        # it will be 'tackle', 'out_of_bounds', 'touchdown', etc...
        elif play['event'].iloc[i] == stop_event:
            return round(dist, 2), round(time/10, 2)
        
        if moving:
            dist += play['dis'].iloc[i]
            time += 1
        #elif not moving and dist > 0:
            # Run has finished
            
            
def yards_after_contact(play):
    
    
    stop_index = play['event'].last_valid_index()
    stop_event = play['event'][stop_index]
    
    if stop_event == 'qb_kneel':
        return np.nan
    
    dist = 0
    moving = False

    for i in range(play.shape[0]):
        
        # We do not know which man 'man_in_motion' refers to
        # Use 'ball_snap' as start of distance
        if play['event'].iloc[i] == 'first_contact':
            moving = True
            
        # final non NA value marks the end of the run
        # it will be 'tackle', 'out_of_bounds', 'touchdown', etc...
        elif play['event'].iloc[i] == stop_event:
            return round(dist, 2)
        
        if moving:
            dist += play['dis'].iloc[i]
            
def max_speed(play):
    
    stop_index = play['event'].last_valid_index()
    stop_event = play['event'][stop_index]
    
    if stop_event == 'qb_kneel':
        return np.nan
    
    speeds = []
    moving = False

    
    for i in range(play.shape[0]):
       
        if play['event'].iloc[i] == 'ball_snap' or play['event'].iloc[i] == 'snap_direct':
            moving = True
            
        # final non NA value marks the end of the run
        # it will be 'tackle', 'out_of_bounds', 'touchdown', etc...
        elif play['event'].iloc[i] == stop_event:
            
            try:
                return max(speeds)
            except:
                # Missing ball snap tag in tracking data
                return np.nan
                
        if moving:
            speeds.append(play['s'].iloc[i])


def contact_speed(play):
    """
    Output: player speed at time of contact
    """

    for i in range(play.shape[0]):
       
        if play['event'].iloc[i] == 'first_contact' or play['event'].iloc[i] == 'tackle':
            return play['s'].iloc[i]
        
    return 'no contact'
            
            
            
            

    



# Iteration procedure

num_plays = df.shape[0]

# Get all file names from directory
all_files = os.listdir()

# Collect tracking data filenames
game_filenames = []

for file in all_files:
    if file.startswith('tracking'):
        game_filenames.append(file)
        
# Create empty new columns:
# - tot_dist
tot_dist_col = pd.Series(np.nan, index = range(num_plays))
tot_time_col = pd.Series(np.nan, index = range(num_plays))
yds_aft_ct_col = pd.Series(np.nan, index = range(num_plays))
max_spd_col = pd.Series(np.nan, index = range(num_plays))
ct_spd_col = pd.Series(np.nan, index = range(num_plays))



# Loop over all tracking files (steps below)
# 1. load next file
# 2. subset
# 3. group by playId
# 4. apply appropriate function
# 5. fill new empty column with values at appropriate index







total_rows = 0
files_read = 0
missing = 0

for filename in game_filenames:
    
    files_read += 1
    
    print("{} files to go".format(len(game_filenames) - files_read))
    
    game = pd.read_csv(filename)
    game['split_name'] = game['displayName'].apply(lambda x: x.split()[0][0].lower() + ' ' + x.split()[1].lower() if len(x.split()) > 1 else x)  

    game_id = int(re.findall(r'\d+', filename)[0])
    print('current gameId: {}'.format(game_id))

    # Subset to keep desired rows associated with this game
    look_up = pd.DataFrame(rush_plays[game_id], columns = ['orig.index', 'playId', 'RunSuccess', 'PlayResult', 'Rusher'])
    look_up['split_name'] = look_up['Rusher'].apply(lambda x: x.split('.')[0][0].lower() + ' ' + x.split('.')[1].lower())
    
    game_rush = pd.merge(look_up, game)
    
    # group by playId and check that only one unique name exists per group
    # detect of two players with similar names were on field during same play
    
    
    rows = game_rush.shape[0]
    total_rows += rows

    grouped_by_playId = game_rush.groupby('playId')
    
    
    
    
    
    #if not tag_check(grouped_by_playId):
        #print('Unbalanced tags in game {}'.format(game_id))
    
    # Collect playId (index) and time to throw for all plays in one game
    d_t = {}
    yds_aft_ct = {}
    mx_spd = {}
    ct_spd = {}
    for name, group in grouped_by_playId:
        d_t[group['playId'].iloc[0]] = tot_dist_time(group)
        yds_aft_ct[group['playId'].iloc[0]] = yards_after_contact(group)
        mx_spd[group['playId'].iloc[0]] = max_speed(group)
        ct_spd[group['playId'].iloc[0]] = contact_speed(group)
    

    for play in rush_plays[game_id]:

        index = play[0]
        play_num = play[1]
        
        #print("index: ", index)
        #print("play num: ", play_num)
        #print("d_t: ", d_t[play_num])
        #print(d_t[play_num][1])
       
    
        try:
            tot_dist_col[index] = d_t[play_num][0]
            tot_time_col[index] = d_t[play_num][1]
            yds_aft_ct_col[index] = yds_aft_ct[play_num]
            max_spd_col[index] = mx_spd[play_num]
            ct_spd_col[index] = ct_spd[play_num]
        except TypeError:
            # QB kneel play; nan values for rush_player
            tot_dist_col[index] = np.nan
            tot_time_col[index] = np.nan
            yds_aft_ct_col[index] = np.nan
            max_spd_col[index] = np.nan
            ct_spd_col[index] = np.nan
        except KeyError:
            missing += 1
            print('Play num {} has been removed from tracking data'.format(play_num))
            continue
        

# Add column
df.insert(loc = 29, column = 'TotalDist', value = tot_dist_col)
df.insert(loc = 30, column = 'TotalTime', value = tot_time_col)
avg_speed = round(df['TotalDist']/df['TotalTime'], 2)

efficiency = round(df['TotalDist']/df['PlayResult'], 2)

df.insert(loc = 31, column = 'AvgSpeed', value = avg_speed)
df.insert(loc = 32, column = 'MaxSpeed', value = max_spd_col)
df.insert(loc = 33, column = 'ContactSpeed', value = ct_spd_col)
df.insert(loc = 34, column = 'YardsAfterContact', value = yds_aft_ct_col)
df.insert(loc = 35, column = 'EFF', value = efficiency)

df = df.replace(np.inf, np.nan)
df = df.replace('no contact', np.nan)

# Save updated version of plays.csv with new columns of calculated values
df.to_csv('plays_v2.csv')


# Recalculate Running Back rankings using new variables
# See how the other variables stack up when sorted by adjusted success rate

success_rate_v2 = df.groupby('Rusher')['RunSuccess'].apply(lambda x: round((x*1).sum()/x.shape[0], 2))

rushes = df.groupby('Rusher')['RunSuccess'].count()
successful = df.groupby('Rusher')['RunSuccess'].apply(lambda x: round((x*1).sum()))
avg_eff = round(df.groupby('Rusher')['EFF'].mean(), 2)
avg_max_spd = round(df.groupby('Rusher')['MaxSpeed'].mean(), 2)
avg_ct_spd = round(df.groupby('Rusher')['ContactSpeed'].mean(), 2)
avg_yds_aft_ct = round(df.groupby('Rusher')['YardsAfterContact'].mean(), 2)

rb_success_rate_v2 = pd.DataFrame({'Rushes': rushes,
                                   'Successful': successful,
                                   'AvgEFF': avg_eff, 
                                   'AvgMaxSpeed': avg_max_spd,
                                   'AvgCtSpeed': avg_ct_spd,
                                   'AvgYardsAfterContact': avg_yds_aft_ct,
                                   'SuccessRate':success_rate})

# sorted by confidence in success rate

rb_success_rate_v2['Conf_Adj_Rank'] = rb_success_rate.apply(lambda x: round(ci_lower_bounds(x['Successful'], x['Rushes'], 0.95), 4), axis = 1)

rb_success_rate_v2.sort_values('Conf_Adj_Rank', inplace = True, ascending = False)
rb_success_rate_v2.to_csv('success_rate_adj_v2.csv')


import seaborn as sns
import matplotlib.pyplot as plt

# Reset default params
sns.set()
sns.set(rc={'figure.figsize':(11.7,8.27)})

# Set context to `"paper"`
sns.set_context("poster")

sns.swarmplot(x="quarter", y="ContactSpeed", data=df[~df['ContactSpeed'].isin(['no contact'])])
plt.show()

sns.set_context("poster")
ax = sns.violinplot(x = "YardsAfterContact", data=df)
ax.set_title("Yards After Contact (rushing plays)")
#grid.set_xticklabels(rotation=30)
#ax = sns.violinplot(x = "YardsAfterContact", data=df)
# Set the `xlim`
#ax.set(xlim=(0, 100))
ax.figure.savefig("yds_ct_dist.png")
ax = sns.regplot(x="ContactSpeed", y="YardsAfterContact", data=df[~df['ContactSpeed'].isin(['no contact'])], fit_reg = False)
ax.set(xlabel='Speed at Moment of Contact (yds/s)', ylabel='Yards Gained After Contact')
ax.figure.savefig("spd_yds_gain.png")
plt.show()

ax.set_context("poster")
ax = sns.regplot(x="MaxSpeed", y="YardsAfterContact", data=df[~df['MaxSpeed'].isin(['no contact'])], fit_reg = False)
ax.set(xlabel='Max Speed (yds/s)', ylabel='Yards Gained After Contact')
ax.figure.savefig("max_spd_yds_gain2.png")
plt.show()

ax.set_context("poster")
ax = sns.regplot(x="AvgSpeed", y="RunSuccess", data=df[df['RunSuccess'] == True].isin(['no contact'])], fit_reg = False)
ax.set(xlabel='Max Speed (yds/s)', ylabel='Yards Gained After Contact')
ax.figure.savefig("max_spd_yds_gain2.png")
plt.show()


from scipy.stats.stats import pearsonr

x = df['ContactSpeed']
y = df['YardsAfterContact']

def r2(x,y):
    nas = nas = np.logical_or(np.isnan(x), np.isnan(y))
    return pearsonr(x[~nas], y[~nas])[0]
sns.set_context("poster")
h = sns.jointplot(x, y, kind="reg", stat_func=r2)
h.set_axis_labels(xlabel='Contact Speed (yds/s)', ylabel='Yards Gained After Contact')
plt.show()
h.savefig("spd_yds_gain_corr.png")


x = df['EFF']
y = df['YardsAfterContact']

def r2(x,y):
    nas = nas = np.logical_or(np.isnan(x), np.isnan(y))
    return pearsonr(x[~nas], y[~nas])[0]
sns.set_context("poster")
h = sns.jointplot(x, y, kind="reg", stat_func=r2)
h.set_axis_labels(xlabel='Rushing Efficiency', ylabel='Yards Gained After Contact')
plt.show()
h.savefig("spd_eff_corr.png")

sns.set_context("poster")
ax = sns.violinplot(x="RunSuccess", y='ContactSpeed', data=df, palette="muted")
ax.figure.savefig("run_spd.png")

sns.set_context("poster")
ax = sns.violinplot(x="RunSuccess", y='YardsAfterContact', data=df, palette="muted")
ax.figure.savefig("run_suc_yds.png")

sns.set_context("poster")
ax = sns.violinplot(x="RunSuccess", y='EFF', data=df, palette="muted")
ax.figure.savefig("run_spd.png")

ax = sns.regplot(x="AvgSpeed", y="EFF", data=df, fit_reg = False)
ax.set(xlabel='Average Speed Throughout A Play (yds/s)', ylabel='Efficiency')

df['MaxSpeed'].describe()
df[df['RunSuccess'] == False]['ContactSpeed'].describe()

df[df['RunSuccess'] == False]['YardsAfterContact'].describe()


success_rate = df.groupby('Rusher')['RunSuccess'].apply(lambda x: round((x*1).sum()/x.shape[0], 2))

max_speed = df.groupby('Rusher')['MaxSpeed'].mean()
avg_spd = df.groupby('Rusher')['AvgSpeed'].mean()

successful = df.groupby('Rusher')['RunSuccess'].apply(lambda x: round((x*1).sum()))

rb_success_rate = pd.DataFrame({'Rushes': rushes, 'Successful': successful, 'MaxSpeed': max_speed, 'AvgSpeed': avg_spd, 'SuccessRate':success_rate})
rb_success_rate.sort_values('SuccessRate', inplace = True, ascending = False)


eff = df.groupby('Rusher')['EFF'].mean()
<file_sep>---
title: "NFL Big Data Bowl 2019"
author: "<NAME> | <EMAIL> | (310) 923-6337"
date: "January 22, 2019"
output:
  pdf_document:
    fig_caption: yes
    keep_tex: yes
fontsize: 12pt

#output: html_document
#runtime: shiny

---




```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = TRUE)
library(kableExtra)
library(reticulate)
```

# Introduction

Speed is exciting. It should be no surprise that many of the metrics that football analysts and fans love to quote have an inherent speed component. Sometimes this fascination with speed is explicit. A good example is the Fastest Ball Carriers metric. More often though speed is simply implied like when ranking quarterbacks by Time to Throw or receivers by Yards After Catch. Regardless of the perspective, speed is a crucial aspect of the game.

In some situations understanding the value of speed is straightforward. For example, it is just as important for wide receivers to have explosive acceleration and high top-end speed as it is for the opposing defensive backs who are charged with disrupting pass routes. Additionally, a quarterback who can fire off a pass quickly also needs receivers who can get to their positions on the field rapidly.

In other situations the way that speed contributes to the success of a play is less obvious. As a result, speed metrics might not immediately seem useful. In order to appreciate the impact of speed we need context, and we can establish that context with data. We already collect a lot of data on NFL players and games in order to gain insights and improve our decisions. Now that we have the ability to take substantially more measurements than ever before during an NFL game, we face an exciting new challenge: how do we find value in the data that is actually useful for teams?

To answer that question we need to think critically about what we should measure, and can actually measure, during a game. Once we properly frame the problems we want to solve, we can formulate meaningful questions that reveal what the collected data can explain.

# The Effectiveness of Speed

There are ways to be effective using speed other than simply being the fastest player to run a 40 yard dash. It is definitely exciting to watch an elite player expertly evade defenders and open up the throttle for an uncontested touchdown. It certainly helps to be the fastest player on the field, but the gametime opportunities to get up to top-speed without interruption are limited. It is more useful to think about speed as a capability. In other words, it is not always top speed, but more often optimal speed and optimal movement that win the day.

If a more general way to think about speed is optimal speed, we need to think about what that means. A combination of various decisive factors such as decision-making, speed, balance, maneuverability, momentum, and strength contribute to the success of a football play. While some factors, like decision-making, might be hard to measure, we can study the outcomes of plays as a means to learn about how these factors might interact. Moreover, with Next Gen Stats, we can specifically dive into the necessary details to learn about what encourages the outcomes we want to reproduce.

# Focus of This Study

In recent NFL seasons rule changes, as well as breakout players, have placed the focus on quarterbacks and wide receiver metrics. We are currently in an era of record-breaking offensive performances. As a result, the focus of metrics and analysis is heavily weighted toward understanding, and often revering, NFL teams with a pass-heavy offense. This study takes an alternate approach and considers optimal speed in the rushing game.

Rushing plays fail for many reasons, and many of those reasons are very difficult, if not impossible, to consistently predict. There is legitimate value in determining why a play did not work. Unfortunately, that can often be a fairly complicated and subjective task. So instead, as a more straightforward initial approach, we will look primarily to plays that were successful at gaining yards. We want to understand the speed-related factors that keep a play alive and help propel a drive to the end zone. We want to know what encourages that success.

To help us find our answer, we will focus on several fundamental themes:

1. Efficiency

2. Momentum

# Data Preparation

The Big Data Bowl plays.csv data is our roadmap. It serves as the guide to plan an analysis of the tracking data files. The goal is to find all running plays and associated ball carriers in order to dive into player-level game data. While each of the data sets could have been merged into one large data set, it is more intuitive to start with a specific purpose and then gather only what is needed from each of the 91 game tracking data sets.

First, a subset of all non-penalty and non-special teams plays is taken. Two-point conversion plays are also removed. While these do contain some rushing plays, they represent a special-case and correspond to only a small portion of the data. Therefore they are not included. For similar reasons, fumble plays are excluded as well.

Once this is done, a column containing the names of the ball carriers on each play is inserted into the play data in order to facilitate pairing ball carriers with plays.


# Theme 1: Efficiency

Players who use speed in an optimal way are efficient. Their decision-making and agility often lead to additional yards when they are most needed. To operationalize the concept of rushing efficiency we will in part use the popular Running Back Success Rate and Rushing Efficiency metrics.

> # Running Back Success Rate (credit: Football Outsiders):
> * A play counts as a "Success" if it gains 40% of yards to go on first down, 60% of yards to go on second down, and 100% of yards to go on third or fourth down.

> * If the team is behind by more than a touchdown in the fourth quarter, the benchmarks switch to 50%/65%/100%.

> * If the team is ahead by any amount in the fourth quarter, the benchmarks switch to 30%/50%/100%.

> To calculate success rate a GainNeed column was first added to the plays.csv data set to represent yards gained divided by yards needed for each rushing play. Then, using a function that incorporated the associated decision rules, a new RunSuccess column was generated to indicate if a run was a deemed a success. Determining success rate then became a simple task of grouping the plays.csv data by ball carrier, and dividing a player's total successful runs by their total carries.

> You could stop here and simply sort players by their success rates. However, this presents a few problems. For one, some rushers appear more frequently in the data than others. This imbalance results in their success rate being more representative since it is based on more data. Additionally, other players have few total carries but many of them are successful. This results in a high success rate based on little evidence.

> If success rate is going to be used as criteria for sorting and ranking running backs then we must weight the ranking with a confidence measure. Think of it like this: at an online retailer's website you naturally trust a 4.5-star rating with 1,000 reviews more than a 5-star rating with 10 reviews. When there is less evidence there is greater uncertainty. In our case, the necessary correction can be made using a clever technique that several websites employ to allow users to sort content by "best."

>> # Running Back Ranking Scheme

>> Since the RunSuccess column only takes on values of true or false, we can create an adjusted ranking scheme using a method known as the Lower bound of a Wilson score confidence interval for a Bernoulli parameter. The method takes a set of positive and negative ratings (e.g., true or false) and estimates the real fraction of positive ratings provided a desired confidence level.

>> While the success rate metric seeks to capture the value of a rushing play, this sorting scheme provides a confidence-adjusted ranking. In Table 1 only rushers with a success rate less than one were included in the top ten. This was done to facilitate recognition of the fact that a small sample size and the successful rush proportion can result in the metric not being very useful. Sorting by success rate alone yields does not yield a useful ranking. However, when this confidence-adjusted success rate ranking scheme is employed (Table 2) note how the top ten rushers are a more representative group in terms of total rushes. While more elaborate ranking schemes exist, usually based on the influence of additional metrics, this approach is a very simple way to add context to a commonly used metric.

>> This approach also has the benefit of including all data as opposed to methods that would simply try to ignore some of the data with a subset and an arbitrary threshold (e.g., only display rusers with more than 50 total rushses).



```{python, echo = FALSE}


import pandas as pd
rb1 = pd.read_csv('success_rate.csv')
rb2 = pd.read_csv('success_rate_adj.csv')


```



```{r, fig.cap='test!', echo=FALSE}


py$rb1[py$rb1['SuccessRate'] < 1.0,][1:10,] %>%
  kable(caption='Non-adjusted running back ranking by success rate') %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)


```


\newpage



```{r, echo=FALSE}


py$rb2[py$rb2['SuccessRate'] < 1.0,][1:10,] %>%
  kable(caption = 'confidence-adjusted ranking by success rate') %>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"), full_width = FALSE)


```

> # Rushing Efficiency (credit: Next Gen Stats):
> * Rushing Efficiency is calculated by taking the total distance a player traveled on rushing plays as a ball carrier according to Next Gen Stats (measured in yards) per rushing yards gained. The lower the number, the more of a North/South runner.

> Rushing efficiency is a metric that serves as a proxy for many aspects of a run play. It is the product of a player's athleticism and agility. It is also reflective of a player's effort to fight and weave through traffic to gain more yards. Ultimately, the rushing efficiency metric attempts to account for all of the unique movements related to a particular position.

> To calculate Rushing Efficiency, or EFF, a column was generated by iterating over the game tracking data sets and calculating a rusher's total distances traveled on all plays. This distance column was then divided by the PlayResult column to derive the EFF column.

\newpage

# Theme 2: Momentum

Rushers who use speed in an optimal way are also tenacious. No matter the situation, they find a way to maintain their momentum until the defense forces them to stop, or until they score a touchdown. Sometimes being a battering ram gets the extra yards. In other situations it is all about the agility necessary to avoid or shed tackles. For some players, given their size and strength, the likelihood that they will gain yards after contact might increase after they accelerate up to a certain optimal speed.

The best way to gauge a player's ability to maintain momentum is to measure the yards they gain after first contact with the defense. Measuring yards gained after contact is a way to identify and understand players who best utilize their remaining speed to make forward progress. Additionally, measuring a player's speed at the moment they make contact is vital to understanding what it means to use speed effectively throughout a rushing play.

> # Yards After Contact
> * Reveals the influence of momentum and maneuvering in gaining additional yards before the play is over.

> # Contact Speed
> * Determined by calculating a rushing ball carrier's speed at the moment they make contact with defensive resistance.

The Yards After Contact metric is derived from each of the run plays a particular ball carrier participated in throughout all of the game tracking data files. For a particular game and play, a ball carrier's data is examined in order to determine the total distance they traveled following first contact with a member of the opposing team. A corresponding contact speed was also extracted from the appropriate game tracking files in a similar fashion.

# Additional Variables

In addition to the variables and columns described, additional variables were calculated or derived and inserted into the plays.csv data set to facilitate answering questions and generating exploratory plots. The entire procedure is detailed and available for review in the master.py Python code file that can be found at the GitHub link at the end of this paper.

\newpage

# Results

We might as well begin with the most-interesting observation. In Figure 1 we see the relationship between a player's speed at the time of contact and the yards gained after contact.

```{python, echo = FALSE}

import matplotlib.pyplot as plt
import numpy as np
from scipy.stats.stats import pearsonr

df = pd.read_csv('plays_v2.csv')


```

```{python, echo = FALSE}

import os
os.environ['QT_QPA_PLATFORM_PLUGIN_PATH'] = 'E:/Apps/Anaconda3/Library/plugins/platforms'

```

```{python, echo = FALSE}

import warnings
import seaborn as sns

x = df['ContactSpeed']
y = df['YardsAfterContact']

def corr(x,y):
    nas = np.logical_or(np.isnan(x), np.isnan(y))
    return pearsonr(x[~nas], y[~nas])[0]
sns.set_context("paper")

warnings.filterwarnings("ignore")

fig1 = sns.jointplot(x, y, kind="reg", stat_func=corr)
fig1.set_axis_labels(xlabel='Contact Speed (yds/s)', ylabel='Yards After Contact')
plt.show()
plt.clf()

```

\begin{center}{Figure 1: Contact Speed vs. Yards After Contact}\end{center}

\newpage

Note that while the correlation is positive, and moderately strong, we still have quite a few observations that clearly exceed the trend. These apparent outliers should not be a surprise. Their presence simply demonstrates that the majority of the time it is difficult for a rusher to escape the defense for a large yardage gain on a play.

So we know a relationship exists between a ball carrier's contact speed and the yards they gain after contact. But contact speed might be slower and less-representative of what a talented rusher is capable of during a play. To see how else speed might influence the yards gained after contact we will look at the max player speed on a play and compare it to the yards gained after contact.

```{python, echo=FALSE}

fig2 = sns.regplot(x="MaxSpeed", y="YardsAfterContact", data=df[~df['MaxSpeed'].isin(['no contact'])], fit_reg = False)
fig2.set(xlabel='Max Speed (yds/s)', ylabel='Yards After Contact')
plt.show()
plt.clf()

```

\begin{center}{Figure 2: Max Speed vs. Yards After Contact}\end{center}

It is pretty clear to see in Figure 2 that there is a trend suggesting that higher max speeds are related to gaining more yards.

\newpage

However, some of these observations appear to be rare events. Attempting to model this relationship with a non-linear approach might not prove very useful.

Nevertheless, as anticipated, speed appears to be at least partially related to gaining additional yards after contact. Next we will look at the role that efficiency plays. At first glance Figure 3 seems to be suggesting that the rushing efficiency metric and yards gained after contact do not have a strong association.

```{python, echo=FALSE}

fig3 = sns.regplot(x='EFF', y='YardsAfterContact', data=df, fit_reg=False)
fig3.set(xlabel='Rushing Efficiency', ylabel='Yards After Contact')
plt.show()
plt.clf()

```

\begin{center}{Figure 3: Rushing Efficiency vs. Yards After Contact}\end{center}

For clarification, remember that the Rushing Efficiency metric actually takes on positive, negative, and zero values. This forces the plot to appear like it does not define a strong linear relationship between Rushing Efficiency and yards gained after contact. But remember that values "near" zero are indicative of an efficient player, so we are not looking for a linear relationship. If you look closely you will see that the diagram is actually making a very clear claim: players with a Rushing Efficiency value near zero consistently gain yards after contact, while those with Rushing efficiency values far from zero do not gain many yards after contact. In other words efficient runners gain more yards after contact with the defense.

Finally, given we know that contact speed is important, we should examine how contact speed is related to whether or not a run is a success. Through a comparison in Figure 4 it is clear to see that, among the observations in this data set, player's with a successful run have higher contact speeds.

```{python, echo=FALSE}

fig4 = sns.violinplot(x="RunSuccess", y='ContactSpeed', data=df, palette="muted")
fig4.set(xlabel='Run Success', ylabel='Contact Speed')
plt.show()
plt.clf()

```

\begin{center}{Figure 4: Run Success vs. Contact Speed}\end{center}

\newpage

# Conclusion and Recommendations

Data should be used to help make better decisions, and discover opportunities to improve efficiency and decision-making in organizations. Given the research agenda and data available in this study it is interesting to see that higher speeds are most definitely correlated with more yards. However, it is far from a perfectly linear relationship. On many plays it is not the highest top speed that matters. Rather, speed varies according to the needs of the situation. This provides at least some support for the idea that player's try to use their speed in an optimal way whether or not they consciously realize it.

Exploring the relationships in the data suggests that successful runs have a higher contact speed when they meet the defense. That higher contact speed is also correlated with yards gained after contact. It is important to remember that while gaining yards after contact is important, gained yards are simply the result of getting other critical aspects of a run play correct.

We note that max speeds on a play have a somewhat non-linear relationship with yards gained after contact. This could potentially be an indication that teams should run more motion plays (e.g., jet motion) in order to allow their rushers to accelerate to a higher speed prior to moving downfield. In that way pre-snap motion might represent much more than simply deception.

Finally, the data seems to suggest that more efficient runners tend to have successful runs.


# Recommendations for Rushers

1. Strive to make runs efficient
2. High contact speed matters
3. Train to get to your max speed quickly

**Note:** This study has been performed with observational data from a portion of one full NFL season. As such these recommendations are merely supported by the provided data but not guaranteed to yield improved performance. Successful execution of any football play depends on a large number of factors that interact in a complex fashion.

\newpage

# Sources

https://www.evanmiller.org/how-not-to-sort-by-average-rating.html

https://nextgenstats.nfl.com/glossary


# Code

https://github.com/bradleynott/NFL-Big-Data-Bowl
<file_sep># NFL-Big-Data-Bowl

23JAN2019

### Purpose

This repository contains my code and write-up for the NFL Big Data Bowl 2019 sponsored by Next Gen Stats.

Per the contest rules I am not allowed to share the data provided for the project, but the workflow has
comments throughout to explain how I went about the arrangment of data and analysis.

There was a lot of low-level event data. Instead of merging all of that data with the higher level tables
I chose to plan my work using the higher level tables. I then worked my way through each lower level event data
file according to that plan.

### Update 24JUN19

Added my explanation for how I interpreted the 'first_contact' tag, and why I made the assumption that it is a reference to the ball carrier. Curiously enough with this data set the play description field/column in the plays.csv file indicated who the ball carrier on each play was but it used an abbreviation. The sensor data identifies players with their full names. With a little cross referencing of the player profile data and some disambiguation work I was able to identify and extract the data for ball carriers on each rushing play.

### Update 27JAN19

Recreated report PDF (nfl_big_data_bowl_Brad_Nott_v3.pdf) using R markdown to correct the low resolution plots. Markdown file incorporates Python and R code. CSV data loaded in markdown file were derived from the raw data using master.py

R markdown file is now the primary source for plotting code (not master.py)

### Update 25JAN19

Added code at line 522 in master.py that recreates the previously constructed running back ranking to include the new variables generated by iterating over the tracking file data. This way you can not only see how the running backs rank according to the previously calculated adjusted success rate metric, but you can also see their individual performance in other areas that might have contributed to their success rate.

For some reason I was not able to eliminate all QB rushes from consideration so the output of this code includes some quarterbacks with rushes and corresponding calcualted data, and some with rushes and no calculated data.

### Note 1:

I orignially planned to examine rushing player data and wide receiver data. However times constraints led me to structure my research question to focus on rushing plays only. You will notice in my Python script that I initially performed subsets to retain pass plays and rushing plays. However, prior to iterating throug all of the tracking data I created a dictionary of games (keys) and play numbers (values) for ONLY non-QB non-fumble rushing plays. You could certainly extend the ideas that I considered to other player positions, but I limited the scope of my analysis to rushing plays only in order to comply with the deadline.

### Note 2:

In the Python script functions and code appear as the analysis requires instead of placing all functions at the beginning or in a separate file to be imported. If anything is confusing just ask. If you have access to the same data and want to use my code, run it a few lines at a time like you might a Jupyter Notebook.


Compute time (single thread) to loop through all low-level event data files(loop at code line 426), subset, and apply
necessary functions was ~3 min.

Total Data size:
- ~2 GB

Data Structure:
- collection of 94 tables extracted from a relational database

Approach overview:
- Subset primary dataframe to collect desired criteria
- Iterate over 91 data files cotaining low-level player tracking (event) data
- Use appropriate criteria to create lookup table for current data file
- Subset and group each data file and apply appropriate functions

Computer:
- Desktop with Intel core i7-8700k and 32 GB RAM
