application_vnd.jupyter.stderr_output_3.png text_plain_output_2.png application_vnd.jupyter.stderr_output_1.png
from pyspark.ml import Pipeline

from pyspark.ml.evaluation import RegressionEvaluator

from pyspark.ml.feature import StringIndexer

from pyspark.ml.feature import VectorAssembler

from pyspark.ml.regression import DecisionTreeRegressor, GBTRegressor

si = StringIndexer(inputCol='store_and_fwd_flag', outputCol='store_and_fwd_flag_si', handleInvalid='skip')
va = VectorAssembler(inputCols=['VendorID', 'day', 'hour', 'passenger_count', 'trip_distance', 'RateCodeID', 'store_and_fwd_flag_si', 'PULocationID', 'DOLocationID', 'payment_type'], outputCol='features')
gbt = GBTRegressor(featuresCol='features', labelCol='total_amount', maxDepth=5, maxIter=5, seed=42)
pipeline = Pipeline(stages=[si, va, gbt])

model = pipeline.fit(train)

predictions = model.transform(test)

evaluator = RegressionEvaluator(labelCol='total_amount', predictionCol='prediction', metricName='rmse')

rmse = evaluator.evaluate(predictions)
print('Root Mean Squared Error (RMSE) on test data = %g' % rmse)