text_plain_output_1.png
from pyspark.sql import SparkSession

from pyspark.sql.types import StringType, IntegerType, StructType, StructField, TimestampType, DoubleType

spark = SparkSession.builder.appName('taxi').getOrCreate()

schema = StructType([StructField('VendorID', IntegerType(), True), StructField('tpep_pickup_datetime', TimestampType(), True), StructField('tpep_dropoff_datetime', TimestampType(), True), StructField('passenger_count', IntegerType(), True), StructField('trip_distance', DoubleType(), True), StructField('RatecodeID', IntegerType(), True), StructField('store_and_fwd_flag', StringType(), True), StructField('PULocationID', IntegerType(), True), StructField('DOLocationID', IntegerType(), True), StructField('payment_type', IntegerType(), True), StructField('fare_amount', DoubleType(), True), StructField('extra', DoubleType(), True), StructField('mta_tax', DoubleType(), True), StructField('tip_amount', DoubleType(), True), StructField('tolls_amount', IntegerType(), True), StructField('improvement_surcharge', DoubleType(), True), StructField('total_amount', DoubleType(), True), StructField('congestion_surcharge', DoubleType(), True)])

df = spark.read.csv('/kaggle/input/newyork-yellow-taxi-trip-data-2020-2019/*2020*', schema=schema, header=True)

len(df.columns)