text_plain_output_1.png
from pyspark.ml.feature import VectorAssembler

from pyspark.sql import SparkSession

from pyspark.sql.functions import col

from pyspark.sql.functions import split

from pyspark.sql.types import IntegerType

import numpy as np
from pyspark import SparkContext, SparkFiles
from pyspark.sql import SparkSession
import string
import matplotlib.pyplot as plt
from pyspark.sql.functions import split
from pyspark.sql.functions import col
from pyspark.sql.types import IntegerType
spark = SparkSession.builder.master('local').appName('NFL').getOrCreate()

playerData = spark.read.csv('../input/nfl-big-data-bowl-2022/players.csv', header=True)
playData = spark.read.csv('../input/nfl-big-data-bowl-2022/plays.csv', header=True)

returnData = playData.filter(playData.kickReturnYardage != 'NA').filter(playData.returnerId != 'NA').drop('gameId', 'playId', 'quarter', 'possessionTeam', 'yardlineSide', 'yardlineNumber', 'gameClock', 'penaltyJerseyNumbers', 'preSnapHomeScore', 'preSnapVisitorScore', 'passResult', 'absoluteYardlineNumber')
reducedPlayerData = playerData.drop('birthDate', 'collegeName', 'Position', 'displayName')
returnData = returnData.withColumnRenamed('returnerId', 'nflId')
merged3 = returnData.join(reducedPlayerData, returnData.nflId == reducedPlayerData.nflId)
merged3 = merged3.filter(merged3.weight != 'NA').filter(merged3.height != 'NA')

new_height = merged3.withColumn('height_feet', split(col('height'), '-').getItem(0)).withColumn('height_inch', split(col('height'), '-').getItem(1))
new_height = new_height.withColumn('height_feet', new_height['height_feet'].cast(IntegerType()))
new_height = new_height.withColumn('height_inch', new_height['height_inch'].cast(IntegerType()))
new_height = new_height.withColumn('weight', new_height['weight'].cast(IntegerType()))
new_height = new_height.withColumn('kickReturnYardage', new_height['kickReturnYardage'].cast(IntegerType()))

new_height = new_height.replace(4, 48, 'height_feet')
new_height = new_height.replace(5, 60, 'height_feet')
new_height = new_height.replace(6, 72, 'height_feet')
new_height = new_height.replace(7, 84, 'height_feet')
new_height = new_height.na.fill(value=0, subset=['height_inch'])
fixedData = new_height.withColumn('totalHeight', col('height_feet') + col('height_inch'))

from pyspark.ml.feature import VectorAssembler
vectorAssembler = VectorAssembler(inputCols=['weight', 'totalHeight'], outputCol='features')
regression_df = vectorAssembler.transform(fixedData)
regression_df = regression_df.select(['features', 'kickReturnYardage'])

regression_df.describe().show()