Dragon Real State -- Price Prediction

import pandas as pd

housing = pd.read_csv("data.csv")

housing.head()  #return top 5 rows

housing.info() #gives information about each column in csv

housing['CHAS'].value_counts() # return each type of value in selected column along with its counts

housing['NOX'].value_counts()

housing.describe() #returns 1. count (ignores null values) 2. Mean 3. Standard Deviation 4. Min and Max values 5. How much percentage of values are less than represented value

%matplotlib inline 
#Shows graphs at present i.e. during execution

import matplotlib.pyplot as plt

housing.hist(bins = 50, figsize=(20,15))

import numpy as np

#for learning purpose as it is available in scikit learn
def split_train_test(data,test_ratio): #test ratio is the part of data that needs to be preserved for testing and is not involved in training
    np.random.seed(42) #It fixes the random shuffled values thus preventing overfitting i.e. leaking test data into training data
    shuffled=np.random.permutation(len(data)) #length of the data is permutated randomly shuffled with indices
    test_set_size = int(len(data)*test_ratio)
    test_indices = shuffled[:test_set_size]
    train_indices = shuffled[test_set_size:]
    return data.iloc[train_indices],data.iloc[test_indices]

#train_set, test_set=split_train_test(housing,0.2)

#print(f"Rows in train set: {len(train_set)}\nRows in test set: {len(test_set)}")

from sklearn.model_selection import train_test_split
train_set, test_set = train_test_split(housing, test_size=0.2, random_state=42)

print(f"Rows in train set: {len(train_set)}\nRows in test set: {len(test_set)}")

from sklearn.model_selection import StratifiedShuffleSplit
split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
for train_index, test_index in  split.split(housing,housing['CHAS']):
    strat_train_set = housing.loc[train_index]
    strat_test_set = housing.loc[test_index]

strat_test_set

strat_test_set['CHAS'].value_counts()

strat_train_set['CHAS'].value_counts()

housing = strat_train_set.copy() #Copying the training data to housing attribute 

corr_matrix = housing.corr()

corr_matrix['MEDV'].sort_values(ascending = False)

from pandas.plotting import scatter_matrix
attributes = ["MEDV","RM","ZN","LSTAT"]
scatter_matrix(housing[attributes], figsize=(12,8))

housing.plot(kind="scatter",x="RM",y="MEDV",alpha=0.8)

housing["TAXRM"] = housing["TAX"]/housing["RM"]

housing["TAXRM"]

housing.head()

corr_matrix = housing.corr()
corr_matrix['MEDV'].sort_values(ascending = False)

housing.plot(kind="scatter",x="TAXRM",y="MEDV",alpha=0.8)

housing = strat_train_set.drop("MEDV", axis=1)
housing_labels = strat_train_set["MEDV"].copy()

#option 1
housing.dropna(subset=["RM"]).shape
# Note: No data in original dataframe is harmed until True attribute is not given.

#option 2
housing.drop("RM",axis=1).shape
# Note: No data in original dataframe is harmed.

median = housing["RM"].median()
housing["RM"].fillna(median)
# Note: No data in original dataframe is harmed.

housing.describe() #Before imputing/filling missing attributes.

from sklearn.impute import SimpleImputer
imputer = SimpleImputer(strategy="median")
imputer.fit(housing)

imputer.statistics_ #Shows each column median

X = imputer.transform(housing)

housing_tr = pd.DataFrame(X, columns=housing.columns) # It is a transformed dataframe without missing values

housing_tr.describe()

from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
my_pipeline = Pipeline([
    ('imputer',SimpleImputer(strategy="median")),
      #.... add as many pipelines as needed
    ('std_scalar',StandardScaler()), 
])

housing_num_tr = my_pipeline.fit_transform(housing_tr)

housing_num_tr   #returns numpy array

housing_num_tr.shape

#from sklearn.linear_model import LinearRegression
#from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor # choosen model
#model = LinearRegression() Discarded model due to heavy error of 22.8
#model = DecisionTreeRegressor()
model = RandomForestRegressor()
model.fit(housing_num_tr, housing_labels)

some_data = housing.iloc[:5]

some_labels = housing_labels.iloc[:5]

prepared_data = my_pipeline.transform(some_data)

model.predict(prepared_data) #Gives pedicted values in the form of an array

list(some_labels) #These are the expected values

from sklearn.metrics import mean_squared_error
housing_predictions = model.predict(housing_num_tr)
#lin_mse = mean_squared_error(housing_labels, housing_predictions) linear mean squared error
#lin_rmse = np.sqrt(lin_mse) root mean squared error
mse = mean_squared_error(housing_labels, housing_predictions) # mean squared error
rmse = np.sqrt(mse) # root mean squared error

rmse #The error is 0. It meams, this model has learned the dataset along with factors like noise. It is called overfitting. Our model needs to follow trend and not noise. Hence, we need to avoid the condition of overfitting or underfitting.

from sklearn.model_selection import cross_val_score
scores = cross_val_score(model, housing_num_tr, housing_labels, scoring="neg_mean_squared_error", cv=10)
rmse_scores = np.sqrt(-scores)

rmse_scores

def print_scores(scores):
    print("Scores: ",scores)
    print("Mean: ",scores.mean())
    print("Standard Deviation: ",scores.std())

print_scores(rmse_scores) #best model = RandomForestRegressor

from joblib import dump,load
dump(model,"DRL.joblib")

X_test = strat_test_set.drop("MEDV", axis=1) #Dropping label
Y_test = strat_test_set["MEDV"].copy()
X_test_prepared = my_pipeline.transform(X_test)
final_predictions = model.predict(X_test_prepared)
final_mse = mean_squared_error(Y_test, final_predictions)
final_rmse = np.sqrt(final_mse)

final_rmse #The lesser the rmse, the better is the model. But it should not be 0.