text_plain_output_2.png application_vnd.jupyter.stderr_output_1.png
from pyspark.sql import SparkSession

from pyspark.sql.functions import col

from pyspark.sql.functions import split

from pyspark.sql.types import IntegerType

import matplotlib.pyplot as plt

import matplotlib.pyplot as plt

import numpy as np

import numpy as np

import numpy as np
from pyspark import SparkContext, SparkFiles
from pyspark.sql import SparkSession
import string
import matplotlib.pyplot as plt
from pyspark.sql.functions import split
from pyspark.sql.functions import col
from pyspark.sql.types import IntegerType
spark = SparkSession.builder.master('local').appName('NFL').getOrCreate()

playerData = spark.read.csv('../input/nfl-big-data-bowl-2022/players.csv', header=True)
playData = spark.read.csv('../input/nfl-big-data-bowl-2022/plays.csv', header=True)

returnData = playData.filter(playData.kickReturnYardage != 'NA').filter(playData.returnerId != 'NA').drop('gameId', 'playId', 'quarter', 'possessionTeam', 'yardlineSide', 'yardlineNumber', 'gameClock', 'penaltyJerseyNumbers', 'preSnapHomeScore', 'preSnapVisitorScore', 'passResult', 'absoluteYardlineNumber')
reducedPlayerData = playerData.drop('birthDate', 'collegeName', 'Position', 'displayName')
returnData = returnData.withColumnRenamed('returnerId', 'nflId')
merged3 = returnData.join(reducedPlayerData, returnData.nflId == reducedPlayerData.nflId)
merged3 = merged3.filter(merged3.weight != 'NA').filter(merged3.height != 'NA')

new_height = merged3.withColumn('height_feet', split(col('height'), '-').getItem(0)).withColumn('height_inch', split(col('height'), '-').getItem(1))
new_height = new_height.withColumn('height_feet', new_height['height_feet'].cast(IntegerType()))
new_height = new_height.withColumn('height_inch', new_height['height_inch'].cast(IntegerType()))
new_height = new_height.withColumn('weight', new_height['weight'].cast(IntegerType()))
new_height = new_height.withColumn('kickReturnYardage', new_height['kickReturnYardage'].cast(IntegerType()))

new_height = new_height.replace(4, 48, 'height_feet')
new_height = new_height.replace(5, 60, 'height_feet')
new_height = new_height.replace(6, 72, 'height_feet')
new_height = new_height.replace(7, 84, 'height_feet')
new_height = new_height.na.fill(value=0, subset=['height_inch'])
fixedData = new_height.withColumn('totalHeight', col('height_feet') + col('height_inch'))

import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import gaussian_kde
x = np.array(fixedData.select('weight').collect())
x = x[np.logical_not(np.isnan(x))]
y = np.array(fixedData.select('kickReturnYardage').collect())
y = y[np.logical_not(np.isnan(y))]
plt.hist(x, bins=30, color='blue')
plt.xlabel('Weight (lbs)', fontsize=16)
plt.ylabel('counts', fontsize=16)
plt.title('Distribution of Weights of Kick Returners', fontsize=16)
plt.show()