image_output_1.png
import matplotlib.pyplot as plt

import pandas as pd

import seaborn as sns

import warnings

import numpy as np
import pandas as pd
import torch
from torch.utils.data import TensorDataset, DataLoader, Dataset
import torch.nn as nn
import torch.nn.functional as F
import torchvision
import torchvision.transforms as transforms
import torch.optim as optim
from torch.optim import lr_scheduler
from sklearn.metrics import accuracy_score
import json
from sklearn.tree import DecisionTreeRegressor
from sklearn import tree
from sklearn import preprocessing
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from sklearn.decomposition import PCA
from sklearn.impute import SimpleImputer
from collections import Counter
from sklearn.preprocessing import LabelEncoder, scale
from sklearn.datasets import load_boston
from sklearn.metrics import r2_score
from sklearn.cross_decomposition import PLSRegression, PLSSVD
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import KFold
from sklearn.model_selection import train_test_split
import torch.utils.data
from sklearn.model_selection import train_test_split
import torch
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
import warnings
warnings.filterwarnings('ignore')
pd.set_option('display.max_columns', 500)

train_csv = pd.read_csv('../input/train.csv', keep_default_na=False)
test_csv = pd.read_csv('../input/test.csv', keep_default_na=False)

def preprocess_data(dataset):
    dataset = dataset.replace('NaN', '')
    for col in list(dataset.columns):
        if col != 'Company ' and col != 'Date':
            dataset[col] = pd.to_numeric(dataset[col])
    dataset = dataset.drop(['ID'], axis=1)
    return dataset

train = preprocess_data(train_csv)
test = preprocess_data(test_csv)

train.dtypes

var = 'Aroon Up'
data = pd.concat([train['Price'], train[var]], axis=1)

corcolm = ['SMA', 'MACD', 'MACD_Hist', 'SlowD', 'FastK', 'RSI',
       'FatD', 'FatK', 'WILLR', 'ADX', 'ADXR', 'PPO', 'MOM', 'BOP',
       'CCI', 'CMO', 'ROC', 'ROCR', 'Aroon Down', 'Aroon Up', 
       'MFI', 'TRIX', 'ULTOSC', 'DX', 'MINUS_DI', 'PLUS_DI', 'MINUS_DM',
       'PLUS_DM', 'Real Lower Band', 'MIDPOINT', 'MIDPRICE', 'SAR', 'ATR', 'Chaikin A/D',
       'ADOSC', 'OBV', 'HT_TRENDLINE', 'LEAD SINE', 'SINE', 'TRENDMODE',
       'DCPERIOD', 'HT_DCPHASE', 'PHASE', 'QUADRATURE', 'Price'];
#correlation matrix
corrmat = train[corcolm].corr()
f, ax = plt.subplots(figsize=(15, 15))
sns.heatmap(corrmat, vmax=.8, square=True);

train = train[corcolm]
total = train.isnull().sum().sort_values(ascending=False)
percent = (train.isnull().sum() / train.isnull().count()).sort_values(ascending=False)
missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])

#box plot overallqual/saleprice
var = 'Fuel_Type'
data = pd.concat([train['Price'], train[var]], axis=1)
f, ax = plt.subplots(figsize=(8, 6))
fig = sns.boxplot(x=var, y="Price", data=data)
fig.axis(ymin=0, ymax=160);

#correlation matrix
corrmat = train.corr()
f, ax = plt.subplots(figsize=(12, 9))
sns.heatmap(corrmat, vmax=.8, square=True);

total = train.isnull().sum().sort_values(ascending=False)
percent = (train.isnull().sum() / train.isnull().count()).sort_values(ascending=False)
missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])

train = train.drop(missing_data[missing_data['Total'] > 36].index, 1)
train = train.drop(train.loc[train['Engine'].isnull()].index)
train = train.drop(train.loc[train['Mileage'].isnull()].index)
train.isnull().sum().max()