text_plain_output_1.png
from pyspark.sql import SparkSession

from pyspark.sql import SparkSession
my_spark = SparkSession.builder.getOrCreate()

file_path = '../input/flights.csv'
flights = my_spark.read.csv(file_path, header=True)
flights.createOrReplaceTempView('flights')

flights = flights.withColumn('duration_hrs', flights.air_time / 60)

flights.toPandas().shape[0]

flights.limit(flights.toPandas().shape[0]).toPandas()['duration_hrs'].hist()