text_plain_output_1.png
from datetime import datetime

from sklearn.linear_model import LinearRegression, Lasso

from sklearn.model_selection import KFold, GridSearchCV

from sklearn.model_selection import train_test_split, cross_val_score

import matplotlib.pyplot as plt

import pandas as pd

import random

import seaborn as sns

def evaluate_preds(train_true_values, train_pred_values, test_true_values, test_pred_values):
    pass
TRAIN_DATASET_PATH = '/kaggle/input/real-estate-price-prediction-moscow/train.csv'
TEST_DATASET_PATH = '/kaggle/input/real-estate-price-prediction-moscow/test.csv'

class DataPreprocessing:
    """Подготовка исходных данных"""
    def __init__(self):
        """Параметры класса"""
        self.medians = None
        self.kitchen_square_quantile = None
    def fit(self, X):
        """Сохранение статистик"""
        self.medians = X.median()
        self.kitchen_square_quantile = X['KitchenSquare'].quantile(0.975)
    def transform(self, X):
        """Трансформация данных"""
        X['Rooms_outlier'] = 0
        X.loc[(X['Rooms'] == 0) | (X['Rooms'] >= 6), 'Rooms_outlier'] = 1
        X.loc[X['Rooms'] == 0, 'Rooms'] = 1
        X.loc[X['Rooms'] >= 6, 'Rooms'] = self.medians['Rooms']
        condition = X['KitchenSquare'].isna() | (X['KitchenSquare'] > 16)
        X.loc[condition, 'KitchenSquare'] = self.medians['KitchenSquare']
        X.loc[X['KitchenSquare'] < 6, 'KitchenSquare'] = 6
        X['HouseFloor_outlier'] = 0
        X.loc[X['HouseFloor'] == 0, 'HouseFloor_outlier'] = 1
        X.loc[X['Floor'] > X['HouseFloor'], 'HouseFloor_outlier'] = 1
        X.loc[X['HouseFloor'] == 0, 'HouseFloor'] = self.medians['HouseFloor']
        floor_outliers = X.loc[X['Floor'] > X['HouseFloor']].index
        X.loc[floor_outliers, 'Floor'] = X.loc[floor_outliers, 'HouseFloor'].apply(lambda x: random.randint(1, x))
        current_year = datetime.now().year
        X['HouseYear_outlier'] = 0
        X.loc[X['HouseYear'] > current_year, 'HouseYear_outlier'] = 1
        X.loc[X['HouseYear'] > current_year, 'HouseYear'] = current_year
        if 'Healthcare_1' in X.columns:
            X.drop('Healthcare_1', axis=1, inplace=True)
        X['LifeSquare_nan'] = X['LifeSquare'].isna() * 1
        condition = X['LifeSquare'].isna() & ~X['Square'].isna() & ~X['KitchenSquare'].isna()
        X.loc[condition, 'LifeSquare'] = X.loc[condition, 'Square'] - X.loc[condition, 'KitchenSquare'] - 10
        X.fillna(self.medians, inplace=True)
        X['TestSquare'] = X['Square'] - X['LifeSquare'] - X['KitchenSquare']
        condition1 = (X['TestSquare'] < 0) | (X['LifeSquare'] < 10) & (X['LifeSquare'] > 0)
        X.loc[condition1, 'LifeSquare'] = X.loc[condition1, 'Square'] - X.loc[condition1, 'KitchenSquare'] - 10
        if 'TestSquare' in X.columns:
            X.drop('TestSquare', axis=1, inplace=True)
        return X

class FeatureGenetator:
    """Генерация новых фич"""
    def __init__(self):
        self.floor_max = None
        self.agg_table_rooms_district = None
        self.agg_table_rooms = None
        self.add_dummies_columns = None
    def fit(self, X, y=None):
        X = X.copy()
        df = X[(X['LifeSquare'] > 0) & (X['LifeSquare'] < 275)]
        df['min_square'] = df['Square']
        df['max_square'] = df['Square']
        df['avg_square'] = df['Square']
        df['min_price'] = df['Price']
        df['max_price'] = df['Price']
        df['avg_price'] = df['Price']
        if y is not None:
            self.agg_table_rooms_district = df.groupby(['DistrictId', 'Rooms'], as_index=False).agg({'Price': 'sum', 'Square': 'sum'}).rename(columns={'Price': 'SumPrice', 'Square': 'SumSquare'})
            self.agg_table_rooms = df.groupby(['Rooms'], as_index=False).agg({'min_square': 'min', 'min_price': 'min', 'max_square': 'max', 'max_price': 'max', 'avg_square': 'mean', 'avg_price': 'mean'})
        if y is not None:
            self.floor_max = df['Floor'].max()
            self.house_year_max = df['HouseYear'].max()
            df = self.floor_to_cat(df)
            df = self.year_to_cat(df)
    def transform(self, X):
        X = self.floor_to_cat(X)
        X = self.year_to_cat(X)
        if self.agg_table_rooms_district is not None:
            X = X.merge(self.agg_table_rooms_district, on=['DistrictId', 'Rooms'], how='left')
        if self.agg_table_rooms is not None:
            X = X.merge(self.agg_table_rooms, on=['Rooms'], how='left')
        if 'Price' in X.columns:
            X['PriceCoeff'] = (X['Price'] - X['min_price']) / (X['max_price'] - X['min_price'])
        else:
            X['PriceCoeff'] = random.random()
        condition = (X['LifeSquare'] < 0) | (X['LifeSquare'] > 275)
        X.loc[condition, 'Square'] = X.loc[condition, 'min_square'] + (X.loc[condition, 'max_square'] - X.loc[condition, 'min_square']) * X.loc[condition, 'PriceCoeff']
        X.loc[condition, 'LifeSquare'] = X.loc[condition, 'Square'] - X.loc[condition, 'KitchenSquare'] - 10
        X['base_price'] = X['SumPrice'] / X['SumSquare']
        X['base_price'].fillna(1000, inplace=True)
        X['SquareCoeff'] = (X['Square'] - X['min_square']) / (X['max_square'] - X['min_square'])
        X = self.SquareCoeff_to_cat(X)
        X = self.KitchenSquare_to_cat(X)
        X = self.HouseFloor_to_cat(X)
        X = self.Social_1_to_cat(X)
        X = self.Social_2_to_cat(X)
        X = self.Social_3_to_cat(X)
        columns_old = set(X.columns.tolist())
        X = pd.get_dummies(X.copy(), columns=['year_cat', 'floor_cat', 'Ecology_2', 'Ecology_3', 'Shops_2', 'SquareCoeff_cat', 'KitchenSquare_cat', 'house_floor_cat', 'Social_1_cat', 'Social_2_cat', 'Social_3_cat'])
        X['Ecology_1'] = X['Ecology_1'] * X['base_price'] * X['Square']
        columns_new = set(X.columns.tolist())
        self.add_dummies_columns = columns_new - columns_old
        for col_name in list(self.add_dummies_columns):
            X[col_name] = X[col_name] * X['base_price'] * X['Square']
        return X
    def Social_1_to_cat(self, X):
        bins = [0, 10, 25, 50, X['Social_1'].max()]
        X['Social_1_cat'] = pd.cut(X['Social_1'], bins=bins, labels=False)
        X['Social_1_cat'].fillna(0, inplace=True)
        return X
    def Social_2_to_cat(self, X):
        bins = [0, 500, 1000, 5000, 10000, X['Social_2'].max()]
        X['Social_2_cat'] = pd.cut(X['Social_2'], bins=bins, labels=False)
        X['Social_2_cat'].fillna(0, inplace=True)
        return X
    def Social_3_to_cat(self, X):
        bins = [0, 3, 20, 60, 100, X['Social_3'].max()]
        X['Social_3_cat'] = pd.cut(X['Social_3'], bins=bins, labels=False)
        X['Social_3_cat'].fillna(0, inplace=True)
        return X
    def KitchenSquare_to_cat(self, X):
        bins = [0, 9, 12, X['KitchenSquare'].max()]
        X['KitchenSquare_cat'] = pd.cut(X['KitchenSquare'], bins=bins, labels=False)
        X['KitchenSquare_cat'].fillna(0, inplace=True)
        return X
    def SquareCoeff_to_cat(self, X):
        bins = [0, 2, 4, 6, 8, 10]
        X['SquareCoeff_cat'] = pd.cut(X['SquareCoeff'] * 10, bins=bins, labels=False)
        X['SquareCoeff_cat'].fillna(0, inplace=True)
        return X
    def shops_to_cat(self, X):
        bins = [0, 2, 6, 16, self.floor_max]
        X['floor_cat'] = pd.cut(X['Shops_1'], bins=bins, labels=False)
        X['floor_cat'].fillna(-1, inplace=True)
        return X
    def HouseFloor_to_cat(self, X):
        bins = [0, 5, 9, 24, 35, X['HouseFloor'].max()]
        X['house_floor_cat'] = pd.cut(X['HouseFloor'], bins=bins, labels=False)
        X['house_floor_cat'].fillna(-1, inplace=True)
        return X
    def floor_to_cat(self, X):
        bins = [0, 3, 5, 9, 15, X['Floor'].max()]
        X['floor_cat'] = pd.cut(X['Floor'], bins=bins, labels=False)
        X['floor_cat'].fillna(-1, inplace=True)
        return X
    def year_to_cat(self, X):
        bins = [0, 1925, 1941, 1945, 1955, 1965, 1985, 1995, 2005, self.house_year_max]
        X['year_cat'] = pd.cut(X['HouseYear'], bins=bins, labels=False)
        X['year_cat'].fillna(-1, inplace=True)
        return X

train_df = pd.read_csv(TRAIN_DATASET_PATH)
test_df = pd.read_csv(TEST_DATASET_PATH)
preprocessor = DataPreprocessing()
preprocessor.fit(train_df)
train_df = preprocessor.transform(train_df)
test_df = preprocessor.transform(test_df)
features_gen = FeatureGenetator()
features_gen.fit(train_df, train_df['Price'])
train_df = features_gen.transform(train_df)
add_dummies_columns_train = features_gen.add_dummies_columns
test_df = features_gen.transform(test_df)
add_dummies_columns_test = features_gen.add_dummies_columns

feature_names_list = list(add_dummies_columns_train) + ['Ecology_1']
target_name = 'Price'
X = train_df[feature_names_list]
y = train_df[target_name]
test_df = test_df[feature_names_list]

model = Lasso(0.05)
model.fit(X_train, y_train)

y_train_preds = model.predict(X_train)
y_test_preds = model.predict(X_test)
evaluate_preds(y_train, y_train_preds, y_test, y_test_preds)

cv_score = cross_val_score(model, X, y, scoring='r2', cv=KFold(n_splits=3, shuffle=True, random_state=34))
cv_score

cv_score.mean()