
# coding: utf-8

# This notebook is the reproduction of [A Very Extensive Sberbank Exploratory Analysis](https://www.kaggle.com/captcalculator/a-very-extensive-sberbank-exploratory-analysis) notebook in python. Here i copied only titles, for more detailed description please refer to original notebook.

# In[ ]:


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
color = sns.color_palette()

get_ipython().run_line_magic('matplotlib', 'inline')

pd.options.mode.chained_assignment = None  # default='warn'
pd.set_option('display.max_columns', 500)


# ## Training Data

# In[ ]:


train_df = pd.read_csv("../input/train.csv", parse_dates=['timestamp'])
train_df['price_doc_log'] = np.log1p(train_df['price_doc'])


# ## Missing Data

# In[ ]:


train_na = (train_df.isnull().sum() / len(train_df)) * 100
train_na = train_na.drop(train_na[train_na == 0].index).sort_values(ascending=False)


# In[ ]:


f, ax = plt.subplots(figsize=(12, 8))
plt.xticks(rotation='90')
sns.barplot(x=train_na.index, y=train_na)
ax.set(title='Percent missing data by feature', ylabel='% missing')


# ## Data Quality Issues

# In[ ]:


# state should be discrete valued between 1 and 4. There is a 33 in it that is cleary a data entry error
# Lets just replace it with the mode.
train_df.loc[train_df['state'] == 33, 'state'] = train_df['state'].mode().iloc[0]

# build_year has an erronus value 20052009. Since its unclear which it should be, let's replace with 2007
train_df.loc[train_df['build_year'] == 20052009, 'build_year'] = 2007


# ## Housing Internal Characteristics

# In[ ]:


internal_chars = ['full_sq', 'life_sq', 'floor', 'max_floor', 'build_year', 'num_room', 'kitch_sq', 'state', 'price_doc']
corrmat = train_df[internal_chars].corr()


# In[ ]:


f, ax = plt.subplots(figsize=(10, 7))
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=.5, annot=True)


# ## Area of Home and Number of Rooms

# In[ ]:


f, ax = plt.subplots(figsize=(10, 7))
plt.scatter(x=train_df['full_sq'], y=train_df['price_doc'], c='r')


# In[ ]:


f, ax = plt.subplots(figsize=(10, 7))
ind = train_df[train_df['full_sq'] > 2000].index
plt.scatter(x=train_df.drop(ind)['full_sq'], y=train_df.drop(ind)['price_doc'], c='r', alpha=0.5)
ax.set(title='Price by area in sq meters', xlabel='Area', ylabel='Price')


# In[ ]:


(train_df['life_sq'] > train_df['full_sq']).sum()


# In[ ]:


f, ax = plt.subplots(figsize=(10, 7))
sns.countplot(x=train_df['num_room'])
ax.set(title='Distribution of room count', xlabel='num_room')


# ## Sale Type

# In[ ]:


#can't plot density graph due to bug in kaggle kernels
#https://github.com/mwaskom/seaborn/issues/1103
#g = sns.FacetGrid(train_df, col="product_type", size=6)
#g.map(sns.kdeplot, "price_doc_log", color="r", shade=True)
#g.add_legend()
#ax.set(ylabel='density')

f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
train_df.drop(train_df['product_type'] == 'Investment')["price_doc_log"].plot.kde(ax=ax[0])
train_df.drop(train_df['product_type'] == 'OwnerOccupier')["price_doc_log"].plot.kde(ax=ax[1])
ax[0].set(xlabel='price_log')
ax[1].set(xlabel='price_log')


# In[ ]:


train_df.groupby('product_type')['price_doc'].median()


# ## Build Year

# In[ ]:


f, ax = plt.subplots(figsize=(12, 8))
plt.xticks(rotation='90')
ind = train_df[(train_df['build_year'] <= 1691) | (train_df['build_year'] >= 2018)].index
by_df = train_df.drop(ind).sort_values(by=['build_year'])
sns.countplot(x=by_df['build_year'])
ax.set(title='Distribution of build year')


# In[ ]:


f, ax = plt.subplots(figsize=(12, 6))
by_price = by_df.groupby('build_year')[['build_year', 'price_doc']].mean()
sns.regplot(x="build_year", y="price_doc", data=by_price, scatter=False, order=3, truncate=True)
plt.plot(by_price['build_year'], by_price['price_doc'], color='r')
ax.set(title='Mean price by year of build')


# ## Timestamp

# In[ ]:


f, ax = plt.subplots(figsize=(12, 6))
ts_df = train_df.groupby('timestamp')[['price_doc']].mean()
#sns.regplot(x="timestamp", y="price_doc", data=ts_df, scatter=False, truncate=True)
plt.plot(ts_df.index, ts_df['price_doc'], color='r', )
ax.set(title='Daily median price over time')


# In[ ]:


import datetime
import matplotlib.dates as mdates
years = mdates.YearLocator()   # every year
yearsFmt = mdates.DateFormatter('%Y')
ts_vc = train_df['timestamp'].value_counts()
f, ax = plt.subplots(figsize=(12, 6))
plt.bar(left=ts_vc.index, height=ts_vc)
ax.xaxis.set_major_locator(years)
ax.xaxis.set_major_formatter(yearsFmt)
ax.set(title='Sales volume over time', ylabel='Number of transactions')


# ## Home State/Material

# In[ ]:


f, ax = plt.subplots(figsize=(12, 8))
ts_df = train_df.groupby(by=[train_df.timestamp.dt.month])[['price_doc']].median()
plt.plot(ts_df.index, ts_df, color='r')
ax.set(title='Price by month of year')


# In[ ]:


f, ax = plt.subplots(figsize=(12, 8))
ind = train_df[train_df['state'].isnull()].index
train_df['price_doc_log10'] = np.log10(train_df['price_doc'])
sns.violinplot(x="state", y="price_doc_log10", data=train_df.drop(ind), inner="box")
# sns.swarmplot(x="state", y="price_doc_log10", data=train_df.dropna(), color="w", alpha=.2);
ax.set(title='Log10 of median price by state of home', xlabel='state', ylabel='log10(price)')


# In[ ]:


train_df.drop(ind).groupby('state')['price_doc'].mean()


# In[ ]:


f, ax = plt.subplots(figsize=(12, 8))
ind = train_df[train_df['material'].isnull()].index
sns.violinplot(x="material", y="price_doc_log", data=train_df.drop(ind), inner="box")
# sns.swarmplot(x="state", y="price_doc_log10", data=train_df.dropna(), color="w", alpha=.2);
ax.set(title='Distribution of price by build material', xlabel='material', ylabel='log(price)')


# In[ ]:


train_df.drop(ind).groupby('material')['price_doc'].median()


# ## Floor of Home

# In[ ]:


f, ax = plt.subplots(figsize=(12, 8))
plt.scatter(x=train_df['floor'], y=train_df['price_doc_log'], c='r', alpha=0.4)
sns.regplot(x="floor", y="price_doc_log", data=train_df, scatter=False, truncate=True)
ax.set(title='Price by floor of home', xlabel='floor', ylabel='log(price)')


# In[ ]:


f, ax = plt.subplots(figsize=(12, 8))
plt.scatter(x=train_df['max_floor'], y=train_df['price_doc_log'], c='r', alpha=0.4)
sns.regplot(x="max_floor", y="price_doc_log", data=train_df, scatter=False, truncate=True)
ax.set(title='Price by max floor of home', xlabel='max_floor', ylabel='log(price)')


# In[ ]:


f, ax = plt.subplots(figsize=(12, 8))
plt.scatter(x=train_df['floor'], y=train_df['max_floor'], c='r', alpha=0.4)
plt.plot([0, 80], [0, 80], color='.5')


# In[ ]:


train_df.loc[train_df['max_floor'] < train_df['floor'], ['id', 'floor','max_floor']].head(20)


# In[ ]:


## Demographic Characteristics


# In[ ]:


demo_vars = ['area_m', 'raion_popul', 'full_all', 'male_f', 'female_f', 'young_all', 'young_female', 
             'work_all', 'work_male', 'work_female', 'price_doc']
corrmat = train_df[demo_vars].corr()


# In[ ]:


f, ax = plt.subplots(figsize=(10, 7))
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=.5, annot=True)


# In[ ]:


train_df['sub_area'].unique().shape[0]


# In[ ]:


train_df['area_km'] = train_df['area_m'] / 1000000
train_df['density'] = train_df['raion_popul'] / train_df['area_km']
f, ax = plt.subplots(figsize=(10, 6))
sa_price = train_df.groupby('sub_area')[['density', 'price_doc']].median()
sns.regplot(x="density", y="price_doc", data=sa_price, scatter=True, truncate=True)
ax.set(title='Median home price by raion population density (people per sq. km)')


# In[ ]:


f, ax = plt.subplots(figsize=(10, 20))
sa_vc = train_df['sub_area'].value_counts()
sa_vc = pd.DataFrame({'sub_area':sa_vc.index, 'count': sa_vc.values})
ax = sns.barplot(x="count", y="sub_area", data=sa_vc, orient="h")
ax.set(title='Number of Transactions by District')
f.tight_layout()


# In[ ]:


train_df['work_share'] = train_df['work_all'] / train_df['raion_popul']
f, ax = plt.subplots(figsize=(12, 6))
sa_price = train_df.groupby('sub_area')[['work_share', 'price_doc']].mean()
sns.regplot(x="work_share", y="price_doc", data=sa_price, scatter=True, order=4, truncate=True)
ax.set(title='District mean home price by share of working age population')


# ## School Characteristics

# In[ ]:


school_chars = ['children_preschool', 'preschool_quota', 'preschool_education_centers_raion', 'children_school', 
                'school_quota', 'school_education_centers_raion', 'school_education_centers_top_20_raion', 
                'university_top_20_raion', 'additional_education_raion', 'additional_education_km', 'university_km', 'price_doc']
corrmat = train_df[school_chars].corr()


# In[ ]:


f, ax = plt.subplots(figsize=(10, 7))
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=.5, annot=True)


# In[ ]:


train_df['university_top_20_raion'].unique()


# In[ ]:


f, ax = plt.subplots(figsize=(12, 8))
sns.stripplot(x="university_top_20_raion", y="price_doc", data=train_df, jitter=True, alpha=.2, color=".8");
sns.boxplot(x="university_top_20_raion", y="price_doc", data=train_df)
ax.set(title='Distribution of home price by # of top universities in Raion', xlabel='university_top_20_raion', 
       ylabel='price_doc')


# ## Cultural/Recreational Characteristics

# In[ ]:


cult_chars = ['sport_objects_raion', 'culture_objects_top_25_raion', 'shopping_centers_raion', 'park_km', 'fitness_km', 
                'swim_pool_km', 'ice_rink_km','stadium_km', 'basketball_km', 'shopping_centers_km', 'big_church_km',
                'church_synagogue_km', 'mosque_km', 'theater_km', 'museum_km', 'exhibition_km', 'catering_km', 'price_doc']
corrmat = train_df[cult_chars].corr()


# In[ ]:


f, ax = plt.subplots(figsize=(12, 7))
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=.5, annot=True)


# In[ ]:


f, ax = plt.subplots(figsize=(10, 6))
so_price = train_df.groupby('sub_area')[['sport_objects_raion', 'price_doc']].median()
sns.regplot(x="sport_objects_raion", y="price_doc", data=so_price, scatter=True, truncate=True)
ax.set(title='Median Raion home price by # of sports objects in Raion')


# In[ ]:


f, ax = plt.subplots(figsize=(10, 6))
co_price = train_df.groupby('sub_area')[['culture_objects_top_25_raion', 'price_doc']].median()
sns.regplot(x="culture_objects_top_25_raion", y="price_doc", data=co_price, scatter=True, truncate=True)
ax.set(title='Median Raion home price by # of sports objects in Raion')


# In[ ]:


train_df.groupby('culture_objects_top_25')['price_doc'].median()


# In[ ]:


f, ax = plt.subplots(figsize=(10, 6))
sns.regplot(x="park_km", y="price_doc", data=train_df, scatter=True, truncate=True, scatter_kws={'color': 'r', 'alpha': .2})
ax.set(title='Median Raion home price by # of sports objects in Raion')


# ## Infrastructure Features

# In[ ]:


inf_features = ['nuclear_reactor_km', 'thermal_power_plant_km', 'power_transmission_line_km', 'incineration_km',
                'water_treatment_km', 'incineration_km', 'railroad_station_walk_km', 'railroad_station_walk_min', 
                'railroad_station_avto_km', 'railroad_station_avto_min', 'public_transport_station_km', 
                'public_transport_station_min_walk', 'water_km', 'mkad_km', 'ttk_km', 'sadovoe_km','bulvar_ring_km',
                'kremlin_km', 'price_doc']
corrmat = train_df[inf_features].corr()


# In[ ]:


f, ax = plt.subplots(figsize=(12, 7))
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=.5, annot=True)


# In[ ]:


f, ax = plt.subplots(figsize=(10, 6))
sns.regplot(x="kremlin_km", y="price_doc", data=train_df, scatter=True, truncate=True, scatter_kws={'color': 'r', 'alpha': .2})
ax.set(title='Home price by distance to Kremlin')


# ## Variable Importance

# In[ ]:


from sklearn.ensemble import RandomForestRegressor
from sklearn.preprocessing import LabelEncoder
X_train = train_df.drop(labels=['timestamp', 'id', 'incineration_raion'], axis=1).dropna()
y_train = X_train['price_doc']
X_train.drop('price_doc', axis=1, inplace=True)
for f in X_train.columns:
    if X_train[f].dtype == 'object':
        lbl = LabelEncoder()
        lbl.fit(X_train[f])
        X_train[f] = lbl.transform(X_train[f])
rf = RandomForestRegressor(random_state=0)
rf = rf.fit(X_train, y_train)


# In[ ]:


fi = list(zip(X_train.columns, rf.feature_importances_))
print('## rf variable importance')
d = [print('## %-40s%s' % (i)) for i in fi[:20]]


# ## Train vs Test Data

# In[ ]:


test_df = pd.read_csv("../input/test.csv", parse_dates=['timestamp'])
test_na = (test_df.isnull().sum() / len(test_df)) * 100
test_na = test_na.drop(test_na[test_na == 0].index).sort_values(ascending=False)


# In[ ]:


f, ax = plt.subplots(figsize=(12, 8))
plt.xticks(rotation='90')
sns.barplot(x=test_na.index, y=test_na)
ax.set(title='Percent missing data by feature', ylabel='% missing')


# In[ ]:


all_data = pd.concat([train_df.drop('price_doc', axis=1), test_df])
all_data['dataset'] = ''
l = len(train_df)
all_data.iloc[:l]['dataset'] = 'train'
all_data.iloc[l:]['dataset'] = 'test'
train_dataset = all_data['dataset'] == 'train'


# In[ ]:


f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
all_data['full_sq_log'] = np.log1p(all_data['full_sq'])
all_data.drop(train_dataset)["full_sq_log"].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)["full_sq_log"].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='full_sq_log')
ax[1].set(title='train', xlabel='full_sq_log')


# In[ ]:


f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
all_data['life_sq_log'] = np.log1p(all_data['life_sq'])
all_data.drop(train_dataset)["life_sq_log"].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)["life_sq_log"].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='life_sq_log')
ax[1].set(title='train', xlabel='life_sq_log')


# In[ ]:


f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
all_data['kitch_sq_log'] = np.log1p(all_data['kitch_sq'])
all_data.drop(train_dataset)["kitch_sq_log"].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)["kitch_sq_log"].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='kitch_sq_log')
ax[1].set(title='train', xlabel='kitch_sq_log')


# In[ ]:


f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
sns.countplot(x=test_df['num_room'], ax=ax[0])
sns.countplot(x=train_df['num_room'], ax=ax[1])
ax[0].set(title='test', xlabel='num_room')
ax[1].set(title='train', xlabel='num_room')


# In[ ]:


f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
all_data.drop(train_dataset)["floor"].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)["floor"].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='floor')
ax[1].set(title='train', xlabel='floor')


# In[ ]:


f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
all_data.drop(train_dataset)["max_floor"].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)["max_floor"].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='max_floor')
ax[1].set(title='train', xlabel='max_floor')


# In[ ]:


f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
ax[0].scatter(x=test_df['floor'], y=test_df['max_floor'], c='r', alpha=0.4)
ax[0].plot([0, 80], [0, 80], color='.5')
ax[1].scatter(x=train_df['floor'], y=train_df['max_floor'], c='r', alpha=0.4)
ax[1].plot([0, 80], [0, 80], color='.5')
ax[0].set(title='test', xlabel='floor', ylabel='max_floor')
ax[1].set(title='train', xlabel='floor', ylabel='max_floor')


# In[ ]:


years = mdates.YearLocator()   # every year
yearsFmt = mdates.DateFormatter('%Y')
ts_vc_train = train_df['timestamp'].value_counts()
ts_vc_test = test_df['timestamp'].value_counts()
f, ax = plt.subplots(figsize=(12, 6))
plt.bar(left=ts_vc_train.index, height=ts_vc_train)
plt.bar(left=ts_vc_test.index, height=ts_vc_test)
ax.xaxis.set_major_locator(years)
ax.xaxis.set_major_formatter(yearsFmt)
ax.set(title='Number of transactions by day', ylabel='count')


# In[ ]:


f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
sns.countplot(x=test_df['product_type'], ax=ax[0])
sns.countplot(x=train_df['product_type'], ax=ax[1])
ax[0].set(title='test', xlabel='product_type')
ax[1].set(title='train', xlabel='product_type')


# In[ ]:


f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
sns.countplot(x=test_df['state'], ax=ax[0])
sns.countplot(x=train_df['state'], ax=ax[1])
ax[0].set(title='test', xlabel='state')
ax[1].set(title='train', xlabel='state')


# In[ ]:


f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
sns.countplot(x=test_df['material'], ax=ax[0])
sns.countplot(x=train_df['material'], ax=ax[1])
ax[0].set(title='test', xlabel='material')
ax[1].set(title='train', xlabel='material')


089819487d7f71f68ff49a4682c0d33864d0f0ba
<mask token>

<mask token>
get_ipython().run_line_magic('matplotlib', 'inline')
<mask token>
pd.set_option('display.max_columns', 500)
<mask token>
plt.xticks(rotation='90')
sns.barplot(x=train_na.index, y=train_na)
ax.set(title='Percent missing data by feature', ylabel='% missing')
<mask token>
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=0.5, annot=True)
<mask token>
plt.scatter(x=train_df['full_sq'], y=train_df['price_doc'], c='r')
<mask token>
plt.scatter(x=train_df.drop(ind)['full_sq'], y=train_df.drop(ind)[
    'price_doc'], c='r', alpha=0.5)
ax.set(title='Price by area in sq meters', xlabel='Area', ylabel='Price')
(train_df['life_sq'] > train_df['full_sq']).sum()
<mask token>
sns.countplot(x=train_df['num_room'])
ax.set(title='Distribution of room count', xlabel='num_room')
<mask token>
train_df.drop(train_df['product_type'] == 'Investment')['price_doc_log'
    ].plot.kde(ax=ax[0])
train_df.drop(train_df['product_type'] == 'OwnerOccupier')['price_doc_log'
    ].plot.kde(ax=ax[1])
ax[0].set(xlabel='price_log')
ax[1].set(xlabel='price_log')
train_df.groupby('product_type')['price_doc'].median()
<mask token>
plt.xticks(rotation='90')
<mask token>
sns.countplot(x=by_df['build_year'])
ax.set(title='Distribution of build year')
<mask token>
sns.regplot(x='build_year', y='price_doc', data=by_price, scatter=False,
    order=3, truncate=True)
plt.plot(by_price['build_year'], by_price['price_doc'], color='r')
ax.set(title='Mean price by year of build')
<mask token>
plt.plot(ts_df.index, ts_df['price_doc'], color='r')
ax.set(title='Daily median price over time')
<mask token>
plt.bar(left=ts_vc.index, height=ts_vc)
ax.xaxis.set_major_locator(years)
ax.xaxis.set_major_formatter(yearsFmt)
ax.set(title='Sales volume over time', ylabel='Number of transactions')
<mask token>
plt.plot(ts_df.index, ts_df, color='r')
ax.set(title='Price by month of year')
<mask token>
sns.violinplot(x='state', y='price_doc_log10', data=train_df.drop(ind),
    inner='box')
ax.set(title='Log10 of median price by state of home', xlabel='state',
    ylabel='log10(price)')
train_df.drop(ind).groupby('state')['price_doc'].mean()
<mask token>
sns.violinplot(x='material', y='price_doc_log', data=train_df.drop(ind),
    inner='box')
ax.set(title='Distribution of price by build material', xlabel='material',
    ylabel='log(price)')
train_df.drop(ind).groupby('material')['price_doc'].median()
<mask token>
plt.scatter(x=train_df['floor'], y=train_df['price_doc_log'], c='r', alpha=0.4)
sns.regplot(x='floor', y='price_doc_log', data=train_df, scatter=False,
    truncate=True)
ax.set(title='Price by floor of home', xlabel='floor', ylabel='log(price)')
<mask token>
plt.scatter(x=train_df['max_floor'], y=train_df['price_doc_log'], c='r',
    alpha=0.4)
sns.regplot(x='max_floor', y='price_doc_log', data=train_df, scatter=False,
    truncate=True)
ax.set(title='Price by max floor of home', xlabel='max_floor', ylabel=
    'log(price)')
<mask token>
plt.scatter(x=train_df['floor'], y=train_df['max_floor'], c='r', alpha=0.4)
plt.plot([0, 80], [0, 80], color='.5')
train_df.loc[train_df['max_floor'] < train_df['floor'], ['id', 'floor',
    'max_floor']].head(20)
<mask token>
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=0.5, annot=True)
train_df['sub_area'].unique().shape[0]
<mask token>
sns.regplot(x='density', y='price_doc', data=sa_price, scatter=True,
    truncate=True)
ax.set(title=
    'Median home price by raion population density (people per sq. km)')
<mask token>
ax.set(title='Number of Transactions by District')
f.tight_layout()
<mask token>
sns.regplot(x='work_share', y='price_doc', data=sa_price, scatter=True,
    order=4, truncate=True)
ax.set(title='District mean home price by share of working age population')
<mask token>
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=0.5, annot=True)
train_df['university_top_20_raion'].unique()
<mask token>
sns.stripplot(x='university_top_20_raion', y='price_doc', data=train_df,
    jitter=True, alpha=0.2, color='.8')
sns.boxplot(x='university_top_20_raion', y='price_doc', data=train_df)
ax.set(title='Distribution of home price by # of top universities in Raion',
    xlabel='university_top_20_raion', ylabel='price_doc')
<mask token>
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=0.5, annot=True)
<mask token>
sns.regplot(x='sport_objects_raion', y='price_doc', data=so_price, scatter=
    True, truncate=True)
ax.set(title='Median Raion home price by # of sports objects in Raion')
<mask token>
sns.regplot(x='culture_objects_top_25_raion', y='price_doc', data=co_price,
    scatter=True, truncate=True)
ax.set(title='Median Raion home price by # of sports objects in Raion')
train_df.groupby('culture_objects_top_25')['price_doc'].median()
<mask token>
sns.regplot(x='park_km', y='price_doc', data=train_df, scatter=True,
    truncate=True, scatter_kws={'color': 'r', 'alpha': 0.2})
ax.set(title='Median Raion home price by # of sports objects in Raion')
<mask token>
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=0.5, annot=True)
<mask token>
sns.regplot(x='kremlin_km', y='price_doc', data=train_df, scatter=True,
    truncate=True, scatter_kws={'color': 'r', 'alpha': 0.2})
ax.set(title='Home price by distance to Kremlin')
<mask token>
X_train.drop('price_doc', axis=1, inplace=True)
for f in X_train.columns:
    if X_train[f].dtype == 'object':
        lbl = LabelEncoder()
        lbl.fit(X_train[f])
        X_train[f] = lbl.transform(X_train[f])
<mask token>
print('## rf variable importance')
<mask token>
plt.xticks(rotation='90')
sns.barplot(x=test_na.index, y=test_na)
ax.set(title='Percent missing data by feature', ylabel='% missing')
<mask token>
all_data.drop(train_dataset)['full_sq_log'].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)['full_sq_log'].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='full_sq_log')
ax[1].set(title='train', xlabel='full_sq_log')
<mask token>
all_data.drop(train_dataset)['life_sq_log'].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)['life_sq_log'].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='life_sq_log')
ax[1].set(title='train', xlabel='life_sq_log')
<mask token>
all_data.drop(train_dataset)['kitch_sq_log'].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)['kitch_sq_log'].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='kitch_sq_log')
ax[1].set(title='train', xlabel='kitch_sq_log')
<mask token>
sns.countplot(x=test_df['num_room'], ax=ax[0])
sns.countplot(x=train_df['num_room'], ax=ax[1])
ax[0].set(title='test', xlabel='num_room')
ax[1].set(title='train', xlabel='num_room')
<mask token>
all_data.drop(train_dataset)['floor'].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)['floor'].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='floor')
ax[1].set(title='train', xlabel='floor')
<mask token>
all_data.drop(train_dataset)['max_floor'].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)['max_floor'].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='max_floor')
ax[1].set(title='train', xlabel='max_floor')
<mask token>
ax[0].scatter(x=test_df['floor'], y=test_df['max_floor'], c='r', alpha=0.4)
ax[0].plot([0, 80], [0, 80], color='.5')
ax[1].scatter(x=train_df['floor'], y=train_df['max_floor'], c='r', alpha=0.4)
ax[1].plot([0, 80], [0, 80], color='.5')
ax[0].set(title='test', xlabel='floor', ylabel='max_floor')
ax[1].set(title='train', xlabel='floor', ylabel='max_floor')
<mask token>
plt.bar(left=ts_vc_train.index, height=ts_vc_train)
plt.bar(left=ts_vc_test.index, height=ts_vc_test)
ax.xaxis.set_major_locator(years)
ax.xaxis.set_major_formatter(yearsFmt)
ax.set(title='Number of transactions by day', ylabel='count')
<mask token>
sns.countplot(x=test_df['product_type'], ax=ax[0])
sns.countplot(x=train_df['product_type'], ax=ax[1])
ax[0].set(title='test', xlabel='product_type')
ax[1].set(title='train', xlabel='product_type')
<mask token>
sns.countplot(x=test_df['state'], ax=ax[0])
sns.countplot(x=train_df['state'], ax=ax[1])
ax[0].set(title='test', xlabel='state')
ax[1].set(title='train', xlabel='state')
<mask token>
sns.countplot(x=test_df['material'], ax=ax[0])
sns.countplot(x=train_df['material'], ax=ax[1])
ax[0].set(title='test', xlabel='material')
ax[1].set(title='train', xlabel='material')

<mask token>
color = sns.color_palette()
get_ipython().run_line_magic('matplotlib', 'inline')
pd.options.mode.chained_assignment = None
pd.set_option('display.max_columns', 500)
train_df = pd.read_csv('../input/train.csv', parse_dates=['timestamp'])
train_df['price_doc_log'] = np.log1p(train_df['price_doc'])
train_na = train_df.isnull().sum() / len(train_df) * 100
train_na = train_na.drop(train_na[train_na == 0].index).sort_values(ascending
    =False)
f, ax = plt.subplots(figsize=(12, 8))
plt.xticks(rotation='90')
sns.barplot(x=train_na.index, y=train_na)
ax.set(title='Percent missing data by feature', ylabel='% missing')
train_df.loc[train_df['state'] == 33, 'state'] = train_df['state'].mode().iloc[
    0]
train_df.loc[train_df['build_year'] == 20052009, 'build_year'] = 2007
internal_chars = ['full_sq', 'life_sq', 'floor', 'max_floor', 'build_year',
    'num_room', 'kitch_sq', 'state', 'price_doc']
corrmat = train_df[internal_chars].corr()
f, ax = plt.subplots(figsize=(10, 7))
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=0.5, annot=True)
f, ax = plt.subplots(figsize=(10, 7))
plt.scatter(x=train_df['full_sq'], y=train_df['price_doc'], c='r')
f, ax = plt.subplots(figsize=(10, 7))
ind = train_df[train_df['full_sq'] > 2000].index
plt.scatter(x=train_df.drop(ind)['full_sq'], y=train_df.drop(ind)[
    'price_doc'], c='r', alpha=0.5)
ax.set(title='Price by area in sq meters', xlabel='Area', ylabel='Price')
(train_df['life_sq'] > train_df['full_sq']).sum()
f, ax = plt.subplots(figsize=(10, 7))
sns.countplot(x=train_df['num_room'])
ax.set(title='Distribution of room count', xlabel='num_room')
f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
train_df.drop(train_df['product_type'] == 'Investment')['price_doc_log'
    ].plot.kde(ax=ax[0])
train_df.drop(train_df['product_type'] == 'OwnerOccupier')['price_doc_log'
    ].plot.kde(ax=ax[1])
ax[0].set(xlabel='price_log')
ax[1].set(xlabel='price_log')
train_df.groupby('product_type')['price_doc'].median()
f, ax = plt.subplots(figsize=(12, 8))
plt.xticks(rotation='90')
ind = train_df[(train_df['build_year'] <= 1691) | (train_df['build_year'] >=
    2018)].index
by_df = train_df.drop(ind).sort_values(by=['build_year'])
sns.countplot(x=by_df['build_year'])
ax.set(title='Distribution of build year')
f, ax = plt.subplots(figsize=(12, 6))
by_price = by_df.groupby('build_year')[['build_year', 'price_doc']].mean()
sns.regplot(x='build_year', y='price_doc', data=by_price, scatter=False,
    order=3, truncate=True)
plt.plot(by_price['build_year'], by_price['price_doc'], color='r')
ax.set(title='Mean price by year of build')
f, ax = plt.subplots(figsize=(12, 6))
ts_df = train_df.groupby('timestamp')[['price_doc']].mean()
plt.plot(ts_df.index, ts_df['price_doc'], color='r')
ax.set(title='Daily median price over time')
<mask token>
years = mdates.YearLocator()
yearsFmt = mdates.DateFormatter('%Y')
ts_vc = train_df['timestamp'].value_counts()
f, ax = plt.subplots(figsize=(12, 6))
plt.bar(left=ts_vc.index, height=ts_vc)
ax.xaxis.set_major_locator(years)
ax.xaxis.set_major_formatter(yearsFmt)
ax.set(title='Sales volume over time', ylabel='Number of transactions')
f, ax = plt.subplots(figsize=(12, 8))
ts_df = train_df.groupby(by=[train_df.timestamp.dt.month])[['price_doc']
    ].median()
plt.plot(ts_df.index, ts_df, color='r')
ax.set(title='Price by month of year')
f, ax = plt.subplots(figsize=(12, 8))
ind = train_df[train_df['state'].isnull()].index
train_df['price_doc_log10'] = np.log10(train_df['price_doc'])
sns.violinplot(x='state', y='price_doc_log10', data=train_df.drop(ind),
    inner='box')
ax.set(title='Log10 of median price by state of home', xlabel='state',
    ylabel='log10(price)')
train_df.drop(ind).groupby('state')['price_doc'].mean()
f, ax = plt.subplots(figsize=(12, 8))
ind = train_df[train_df['material'].isnull()].index
sns.violinplot(x='material', y='price_doc_log', data=train_df.drop(ind),
    inner='box')
ax.set(title='Distribution of price by build material', xlabel='material',
    ylabel='log(price)')
train_df.drop(ind).groupby('material')['price_doc'].median()
f, ax = plt.subplots(figsize=(12, 8))
plt.scatter(x=train_df['floor'], y=train_df['price_doc_log'], c='r', alpha=0.4)
sns.regplot(x='floor', y='price_doc_log', data=train_df, scatter=False,
    truncate=True)
ax.set(title='Price by floor of home', xlabel='floor', ylabel='log(price)')
f, ax = plt.subplots(figsize=(12, 8))
plt.scatter(x=train_df['max_floor'], y=train_df['price_doc_log'], c='r',
    alpha=0.4)
sns.regplot(x='max_floor', y='price_doc_log', data=train_df, scatter=False,
    truncate=True)
ax.set(title='Price by max floor of home', xlabel='max_floor', ylabel=
    'log(price)')
f, ax = plt.subplots(figsize=(12, 8))
plt.scatter(x=train_df['floor'], y=train_df['max_floor'], c='r', alpha=0.4)
plt.plot([0, 80], [0, 80], color='.5')
train_df.loc[train_df['max_floor'] < train_df['floor'], ['id', 'floor',
    'max_floor']].head(20)
demo_vars = ['area_m', 'raion_popul', 'full_all', 'male_f', 'female_f',
    'young_all', 'young_female', 'work_all', 'work_male', 'work_female',
    'price_doc']
corrmat = train_df[demo_vars].corr()
f, ax = plt.subplots(figsize=(10, 7))
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=0.5, annot=True)
train_df['sub_area'].unique().shape[0]
train_df['area_km'] = train_df['area_m'] / 1000000
train_df['density'] = train_df['raion_popul'] / train_df['area_km']
f, ax = plt.subplots(figsize=(10, 6))
sa_price = train_df.groupby('sub_area')[['density', 'price_doc']].median()
sns.regplot(x='density', y='price_doc', data=sa_price, scatter=True,
    truncate=True)
ax.set(title=
    'Median home price by raion population density (people per sq. km)')
f, ax = plt.subplots(figsize=(10, 20))
sa_vc = train_df['sub_area'].value_counts()
sa_vc = pd.DataFrame({'sub_area': sa_vc.index, 'count': sa_vc.values})
ax = sns.barplot(x='count', y='sub_area', data=sa_vc, orient='h')
ax.set(title='Number of Transactions by District')
f.tight_layout()
train_df['work_share'] = train_df['work_all'] / train_df['raion_popul']
f, ax = plt.subplots(figsize=(12, 6))
sa_price = train_df.groupby('sub_area')[['work_share', 'price_doc']].mean()
sns.regplot(x='work_share', y='price_doc', data=sa_price, scatter=True,
    order=4, truncate=True)
ax.set(title='District mean home price by share of working age population')
school_chars = ['children_preschool', 'preschool_quota',
    'preschool_education_centers_raion', 'children_school', 'school_quota',
    'school_education_centers_raion',
    'school_education_centers_top_20_raion', 'university_top_20_raion',
    'additional_education_raion', 'additional_education_km',
    'university_km', 'price_doc']
corrmat = train_df[school_chars].corr()
f, ax = plt.subplots(figsize=(10, 7))
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=0.5, annot=True)
train_df['university_top_20_raion'].unique()
f, ax = plt.subplots(figsize=(12, 8))
sns.stripplot(x='university_top_20_raion', y='price_doc', data=train_df,
    jitter=True, alpha=0.2, color='.8')
sns.boxplot(x='university_top_20_raion', y='price_doc', data=train_df)
ax.set(title='Distribution of home price by # of top universities in Raion',
    xlabel='university_top_20_raion', ylabel='price_doc')
cult_chars = ['sport_objects_raion', 'culture_objects_top_25_raion',
    'shopping_centers_raion', 'park_km', 'fitness_km', 'swim_pool_km',
    'ice_rink_km', 'stadium_km', 'basketball_km', 'shopping_centers_km',
    'big_church_km', 'church_synagogue_km', 'mosque_km', 'theater_km',
    'museum_km', 'exhibition_km', 'catering_km', 'price_doc']
corrmat = train_df[cult_chars].corr()
f, ax = plt.subplots(figsize=(12, 7))
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=0.5, annot=True)
f, ax = plt.subplots(figsize=(10, 6))
so_price = train_df.groupby('sub_area')[['sport_objects_raion', 'price_doc']
    ].median()
sns.regplot(x='sport_objects_raion', y='price_doc', data=so_price, scatter=
    True, truncate=True)
ax.set(title='Median Raion home price by # of sports objects in Raion')
f, ax = plt.subplots(figsize=(10, 6))
co_price = train_df.groupby('sub_area')[['culture_objects_top_25_raion',
    'price_doc']].median()
sns.regplot(x='culture_objects_top_25_raion', y='price_doc', data=co_price,
    scatter=True, truncate=True)
ax.set(title='Median Raion home price by # of sports objects in Raion')
train_df.groupby('culture_objects_top_25')['price_doc'].median()
f, ax = plt.subplots(figsize=(10, 6))
sns.regplot(x='park_km', y='price_doc', data=train_df, scatter=True,
    truncate=True, scatter_kws={'color': 'r', 'alpha': 0.2})
ax.set(title='Median Raion home price by # of sports objects in Raion')
inf_features = ['nuclear_reactor_km', 'thermal_power_plant_km',
    'power_transmission_line_km', 'incineration_km', 'water_treatment_km',
    'incineration_km', 'railroad_station_walk_km',
    'railroad_station_walk_min', 'railroad_station_avto_km',
    'railroad_station_avto_min', 'public_transport_station_km',
    'public_transport_station_min_walk', 'water_km', 'mkad_km', 'ttk_km',
    'sadovoe_km', 'bulvar_ring_km', 'kremlin_km', 'price_doc']
corrmat = train_df[inf_features].corr()
f, ax = plt.subplots(figsize=(12, 7))
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=0.5, annot=True)
f, ax = plt.subplots(figsize=(10, 6))
sns.regplot(x='kremlin_km', y='price_doc', data=train_df, scatter=True,
    truncate=True, scatter_kws={'color': 'r', 'alpha': 0.2})
ax.set(title='Home price by distance to Kremlin')
<mask token>
X_train = train_df.drop(labels=['timestamp', 'id', 'incineration_raion'],
    axis=1).dropna()
y_train = X_train['price_doc']
X_train.drop('price_doc', axis=1, inplace=True)
for f in X_train.columns:
    if X_train[f].dtype == 'object':
        lbl = LabelEncoder()
        lbl.fit(X_train[f])
        X_train[f] = lbl.transform(X_train[f])
rf = RandomForestRegressor(random_state=0)
rf = rf.fit(X_train, y_train)
fi = list(zip(X_train.columns, rf.feature_importances_))
print('## rf variable importance')
d = [print('## %-40s%s' % i) for i in fi[:20]]
test_df = pd.read_csv('../input/test.csv', parse_dates=['timestamp'])
test_na = test_df.isnull().sum() / len(test_df) * 100
test_na = test_na.drop(test_na[test_na == 0].index).sort_values(ascending=False
    )
f, ax = plt.subplots(figsize=(12, 8))
plt.xticks(rotation='90')
sns.barplot(x=test_na.index, y=test_na)
ax.set(title='Percent missing data by feature', ylabel='% missing')
all_data = pd.concat([train_df.drop('price_doc', axis=1), test_df])
all_data['dataset'] = ''
l = len(train_df)
all_data.iloc[:l]['dataset'] = 'train'
all_data.iloc[l:]['dataset'] = 'test'
train_dataset = all_data['dataset'] == 'train'
f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
all_data['full_sq_log'] = np.log1p(all_data['full_sq'])
all_data.drop(train_dataset)['full_sq_log'].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)['full_sq_log'].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='full_sq_log')
ax[1].set(title='train', xlabel='full_sq_log')
f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
all_data['life_sq_log'] = np.log1p(all_data['life_sq'])
all_data.drop(train_dataset)['life_sq_log'].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)['life_sq_log'].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='life_sq_log')
ax[1].set(title='train', xlabel='life_sq_log')
f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
all_data['kitch_sq_log'] = np.log1p(all_data['kitch_sq'])
all_data.drop(train_dataset)['kitch_sq_log'].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)['kitch_sq_log'].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='kitch_sq_log')
ax[1].set(title='train', xlabel='kitch_sq_log')
f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
sns.countplot(x=test_df['num_room'], ax=ax[0])
sns.countplot(x=train_df['num_room'], ax=ax[1])
ax[0].set(title='test', xlabel='num_room')
ax[1].set(title='train', xlabel='num_room')
f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
all_data.drop(train_dataset)['floor'].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)['floor'].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='floor')
ax[1].set(title='train', xlabel='floor')
f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
all_data.drop(train_dataset)['max_floor'].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)['max_floor'].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='max_floor')
ax[1].set(title='train', xlabel='max_floor')
f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
ax[0].scatter(x=test_df['floor'], y=test_df['max_floor'], c='r', alpha=0.4)
ax[0].plot([0, 80], [0, 80], color='.5')
ax[1].scatter(x=train_df['floor'], y=train_df['max_floor'], c='r', alpha=0.4)
ax[1].plot([0, 80], [0, 80], color='.5')
ax[0].set(title='test', xlabel='floor', ylabel='max_floor')
ax[1].set(title='train', xlabel='floor', ylabel='max_floor')
years = mdates.YearLocator()
yearsFmt = mdates.DateFormatter('%Y')
ts_vc_train = train_df['timestamp'].value_counts()
ts_vc_test = test_df['timestamp'].value_counts()
f, ax = plt.subplots(figsize=(12, 6))
plt.bar(left=ts_vc_train.index, height=ts_vc_train)
plt.bar(left=ts_vc_test.index, height=ts_vc_test)
ax.xaxis.set_major_locator(years)
ax.xaxis.set_major_formatter(yearsFmt)
ax.set(title='Number of transactions by day', ylabel='count')
f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
sns.countplot(x=test_df['product_type'], ax=ax[0])
sns.countplot(x=train_df['product_type'], ax=ax[1])
ax[0].set(title='test', xlabel='product_type')
ax[1].set(title='train', xlabel='product_type')
f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
sns.countplot(x=test_df['state'], ax=ax[0])
sns.countplot(x=train_df['state'], ax=ax[1])
ax[0].set(title='test', xlabel='state')
ax[1].set(title='train', xlabel='state')
f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
sns.countplot(x=test_df['material'], ax=ax[0])
sns.countplot(x=train_df['material'], ax=ax[1])
ax[0].set(title='test', xlabel='material')
ax[1].set(title='train', xlabel='material')

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
color = sns.color_palette()
get_ipython().run_line_magic('matplotlib', 'inline')
pd.options.mode.chained_assignment = None
pd.set_option('display.max_columns', 500)
train_df = pd.read_csv('../input/train.csv', parse_dates=['timestamp'])
train_df['price_doc_log'] = np.log1p(train_df['price_doc'])
train_na = train_df.isnull().sum() / len(train_df) * 100
train_na = train_na.drop(train_na[train_na == 0].index).sort_values(ascending
    =False)
f, ax = plt.subplots(figsize=(12, 8))
plt.xticks(rotation='90')
sns.barplot(x=train_na.index, y=train_na)
ax.set(title='Percent missing data by feature', ylabel='% missing')
train_df.loc[train_df['state'] == 33, 'state'] = train_df['state'].mode().iloc[
    0]
train_df.loc[train_df['build_year'] == 20052009, 'build_year'] = 2007
internal_chars = ['full_sq', 'life_sq', 'floor', 'max_floor', 'build_year',
    'num_room', 'kitch_sq', 'state', 'price_doc']
corrmat = train_df[internal_chars].corr()
f, ax = plt.subplots(figsize=(10, 7))
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=0.5, annot=True)
f, ax = plt.subplots(figsize=(10, 7))
plt.scatter(x=train_df['full_sq'], y=train_df['price_doc'], c='r')
f, ax = plt.subplots(figsize=(10, 7))
ind = train_df[train_df['full_sq'] > 2000].index
plt.scatter(x=train_df.drop(ind)['full_sq'], y=train_df.drop(ind)[
    'price_doc'], c='r', alpha=0.5)
ax.set(title='Price by area in sq meters', xlabel='Area', ylabel='Price')
(train_df['life_sq'] > train_df['full_sq']).sum()
f, ax = plt.subplots(figsize=(10, 7))
sns.countplot(x=train_df['num_room'])
ax.set(title='Distribution of room count', xlabel='num_room')
f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
train_df.drop(train_df['product_type'] == 'Investment')['price_doc_log'
    ].plot.kde(ax=ax[0])
train_df.drop(train_df['product_type'] == 'OwnerOccupier')['price_doc_log'
    ].plot.kde(ax=ax[1])
ax[0].set(xlabel='price_log')
ax[1].set(xlabel='price_log')
train_df.groupby('product_type')['price_doc'].median()
f, ax = plt.subplots(figsize=(12, 8))
plt.xticks(rotation='90')
ind = train_df[(train_df['build_year'] <= 1691) | (train_df['build_year'] >=
    2018)].index
by_df = train_df.drop(ind).sort_values(by=['build_year'])
sns.countplot(x=by_df['build_year'])
ax.set(title='Distribution of build year')
f, ax = plt.subplots(figsize=(12, 6))
by_price = by_df.groupby('build_year')[['build_year', 'price_doc']].mean()
sns.regplot(x='build_year', y='price_doc', data=by_price, scatter=False,
    order=3, truncate=True)
plt.plot(by_price['build_year'], by_price['price_doc'], color='r')
ax.set(title='Mean price by year of build')
f, ax = plt.subplots(figsize=(12, 6))
ts_df = train_df.groupby('timestamp')[['price_doc']].mean()
plt.plot(ts_df.index, ts_df['price_doc'], color='r')
ax.set(title='Daily median price over time')
import datetime
import matplotlib.dates as mdates
years = mdates.YearLocator()
yearsFmt = mdates.DateFormatter('%Y')
ts_vc = train_df['timestamp'].value_counts()
f, ax = plt.subplots(figsize=(12, 6))
plt.bar(left=ts_vc.index, height=ts_vc)
ax.xaxis.set_major_locator(years)
ax.xaxis.set_major_formatter(yearsFmt)
ax.set(title='Sales volume over time', ylabel='Number of transactions')
f, ax = plt.subplots(figsize=(12, 8))
ts_df = train_df.groupby(by=[train_df.timestamp.dt.month])[['price_doc']
    ].median()
plt.plot(ts_df.index, ts_df, color='r')
ax.set(title='Price by month of year')
f, ax = plt.subplots(figsize=(12, 8))
ind = train_df[train_df['state'].isnull()].index
train_df['price_doc_log10'] = np.log10(train_df['price_doc'])
sns.violinplot(x='state', y='price_doc_log10', data=train_df.drop(ind),
    inner='box')
ax.set(title='Log10 of median price by state of home', xlabel='state',
    ylabel='log10(price)')
train_df.drop(ind).groupby('state')['price_doc'].mean()
f, ax = plt.subplots(figsize=(12, 8))
ind = train_df[train_df['material'].isnull()].index
sns.violinplot(x='material', y='price_doc_log', data=train_df.drop(ind),
    inner='box')
ax.set(title='Distribution of price by build material', xlabel='material',
    ylabel='log(price)')
train_df.drop(ind).groupby('material')['price_doc'].median()
f, ax = plt.subplots(figsize=(12, 8))
plt.scatter(x=train_df['floor'], y=train_df['price_doc_log'], c='r', alpha=0.4)
sns.regplot(x='floor', y='price_doc_log', data=train_df, scatter=False,
    truncate=True)
ax.set(title='Price by floor of home', xlabel='floor', ylabel='log(price)')
f, ax = plt.subplots(figsize=(12, 8))
plt.scatter(x=train_df['max_floor'], y=train_df['price_doc_log'], c='r',
    alpha=0.4)
sns.regplot(x='max_floor', y='price_doc_log', data=train_df, scatter=False,
    truncate=True)
ax.set(title='Price by max floor of home', xlabel='max_floor', ylabel=
    'log(price)')
f, ax = plt.subplots(figsize=(12, 8))
plt.scatter(x=train_df['floor'], y=train_df['max_floor'], c='r', alpha=0.4)
plt.plot([0, 80], [0, 80], color='.5')
train_df.loc[train_df['max_floor'] < train_df['floor'], ['id', 'floor',
    'max_floor']].head(20)
demo_vars = ['area_m', 'raion_popul', 'full_all', 'male_f', 'female_f',
    'young_all', 'young_female', 'work_all', 'work_male', 'work_female',
    'price_doc']
corrmat = train_df[demo_vars].corr()
f, ax = plt.subplots(figsize=(10, 7))
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=0.5, annot=True)
train_df['sub_area'].unique().shape[0]
train_df['area_km'] = train_df['area_m'] / 1000000
train_df['density'] = train_df['raion_popul'] / train_df['area_km']
f, ax = plt.subplots(figsize=(10, 6))
sa_price = train_df.groupby('sub_area')[['density', 'price_doc']].median()
sns.regplot(x='density', y='price_doc', data=sa_price, scatter=True,
    truncate=True)
ax.set(title=
    'Median home price by raion population density (people per sq. km)')
f, ax = plt.subplots(figsize=(10, 20))
sa_vc = train_df['sub_area'].value_counts()
sa_vc = pd.DataFrame({'sub_area': sa_vc.index, 'count': sa_vc.values})
ax = sns.barplot(x='count', y='sub_area', data=sa_vc, orient='h')
ax.set(title='Number of Transactions by District')
f.tight_layout()
train_df['work_share'] = train_df['work_all'] / train_df['raion_popul']
f, ax = plt.subplots(figsize=(12, 6))
sa_price = train_df.groupby('sub_area')[['work_share', 'price_doc']].mean()
sns.regplot(x='work_share', y='price_doc', data=sa_price, scatter=True,
    order=4, truncate=True)
ax.set(title='District mean home price by share of working age population')
school_chars = ['children_preschool', 'preschool_quota',
    'preschool_education_centers_raion', 'children_school', 'school_quota',
    'school_education_centers_raion',
    'school_education_centers_top_20_raion', 'university_top_20_raion',
    'additional_education_raion', 'additional_education_km',
    'university_km', 'price_doc']
corrmat = train_df[school_chars].corr()
f, ax = plt.subplots(figsize=(10, 7))
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=0.5, annot=True)
train_df['university_top_20_raion'].unique()
f, ax = plt.subplots(figsize=(12, 8))
sns.stripplot(x='university_top_20_raion', y='price_doc', data=train_df,
    jitter=True, alpha=0.2, color='.8')
sns.boxplot(x='university_top_20_raion', y='price_doc', data=train_df)
ax.set(title='Distribution of home price by # of top universities in Raion',
    xlabel='university_top_20_raion', ylabel='price_doc')
cult_chars = ['sport_objects_raion', 'culture_objects_top_25_raion',
    'shopping_centers_raion', 'park_km', 'fitness_km', 'swim_pool_km',
    'ice_rink_km', 'stadium_km', 'basketball_km', 'shopping_centers_km',
    'big_church_km', 'church_synagogue_km', 'mosque_km', 'theater_km',
    'museum_km', 'exhibition_km', 'catering_km', 'price_doc']
corrmat = train_df[cult_chars].corr()
f, ax = plt.subplots(figsize=(12, 7))
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=0.5, annot=True)
f, ax = plt.subplots(figsize=(10, 6))
so_price = train_df.groupby('sub_area')[['sport_objects_raion', 'price_doc']
    ].median()
sns.regplot(x='sport_objects_raion', y='price_doc', data=so_price, scatter=
    True, truncate=True)
ax.set(title='Median Raion home price by # of sports objects in Raion')
f, ax = plt.subplots(figsize=(10, 6))
co_price = train_df.groupby('sub_area')[['culture_objects_top_25_raion',
    'price_doc']].median()
sns.regplot(x='culture_objects_top_25_raion', y='price_doc', data=co_price,
    scatter=True, truncate=True)
ax.set(title='Median Raion home price by # of sports objects in Raion')
train_df.groupby('culture_objects_top_25')['price_doc'].median()
f, ax = plt.subplots(figsize=(10, 6))
sns.regplot(x='park_km', y='price_doc', data=train_df, scatter=True,
    truncate=True, scatter_kws={'color': 'r', 'alpha': 0.2})
ax.set(title='Median Raion home price by # of sports objects in Raion')
inf_features = ['nuclear_reactor_km', 'thermal_power_plant_km',
    'power_transmission_line_km', 'incineration_km', 'water_treatment_km',
    'incineration_km', 'railroad_station_walk_km',
    'railroad_station_walk_min', 'railroad_station_avto_km',
    'railroad_station_avto_min', 'public_transport_station_km',
    'public_transport_station_min_walk', 'water_km', 'mkad_km', 'ttk_km',
    'sadovoe_km', 'bulvar_ring_km', 'kremlin_km', 'price_doc']
corrmat = train_df[inf_features].corr()
f, ax = plt.subplots(figsize=(12, 7))
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=0.5, annot=True)
f, ax = plt.subplots(figsize=(10, 6))
sns.regplot(x='kremlin_km', y='price_doc', data=train_df, scatter=True,
    truncate=True, scatter_kws={'color': 'r', 'alpha': 0.2})
ax.set(title='Home price by distance to Kremlin')
from sklearn.ensemble import RandomForestRegressor
from sklearn.preprocessing import LabelEncoder
X_train = train_df.drop(labels=['timestamp', 'id', 'incineration_raion'],
    axis=1).dropna()
y_train = X_train['price_doc']
X_train.drop('price_doc', axis=1, inplace=True)
for f in X_train.columns:
    if X_train[f].dtype == 'object':
        lbl = LabelEncoder()
        lbl.fit(X_train[f])
        X_train[f] = lbl.transform(X_train[f])
rf = RandomForestRegressor(random_state=0)
rf = rf.fit(X_train, y_train)
fi = list(zip(X_train.columns, rf.feature_importances_))
print('## rf variable importance')
d = [print('## %-40s%s' % i) for i in fi[:20]]
test_df = pd.read_csv('../input/test.csv', parse_dates=['timestamp'])
test_na = test_df.isnull().sum() / len(test_df) * 100
test_na = test_na.drop(test_na[test_na == 0].index).sort_values(ascending=False
    )
f, ax = plt.subplots(figsize=(12, 8))
plt.xticks(rotation='90')
sns.barplot(x=test_na.index, y=test_na)
ax.set(title='Percent missing data by feature', ylabel='% missing')
all_data = pd.concat([train_df.drop('price_doc', axis=1), test_df])
all_data['dataset'] = ''
l = len(train_df)
all_data.iloc[:l]['dataset'] = 'train'
all_data.iloc[l:]['dataset'] = 'test'
train_dataset = all_data['dataset'] == 'train'
f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
all_data['full_sq_log'] = np.log1p(all_data['full_sq'])
all_data.drop(train_dataset)['full_sq_log'].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)['full_sq_log'].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='full_sq_log')
ax[1].set(title='train', xlabel='full_sq_log')
f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
all_data['life_sq_log'] = np.log1p(all_data['life_sq'])
all_data.drop(train_dataset)['life_sq_log'].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)['life_sq_log'].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='life_sq_log')
ax[1].set(title='train', xlabel='life_sq_log')
f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
all_data['kitch_sq_log'] = np.log1p(all_data['kitch_sq'])
all_data.drop(train_dataset)['kitch_sq_log'].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)['kitch_sq_log'].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='kitch_sq_log')
ax[1].set(title='train', xlabel='kitch_sq_log')
f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
sns.countplot(x=test_df['num_room'], ax=ax[0])
sns.countplot(x=train_df['num_room'], ax=ax[1])
ax[0].set(title='test', xlabel='num_room')
ax[1].set(title='train', xlabel='num_room')
f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
all_data.drop(train_dataset)['floor'].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)['floor'].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='floor')
ax[1].set(title='train', xlabel='floor')
f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
all_data.drop(train_dataset)['max_floor'].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)['max_floor'].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='max_floor')
ax[1].set(title='train', xlabel='max_floor')
f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
ax[0].scatter(x=test_df['floor'], y=test_df['max_floor'], c='r', alpha=0.4)
ax[0].plot([0, 80], [0, 80], color='.5')
ax[1].scatter(x=train_df['floor'], y=train_df['max_floor'], c='r', alpha=0.4)
ax[1].plot([0, 80], [0, 80], color='.5')
ax[0].set(title='test', xlabel='floor', ylabel='max_floor')
ax[1].set(title='train', xlabel='floor', ylabel='max_floor')
years = mdates.YearLocator()
yearsFmt = mdates.DateFormatter('%Y')
ts_vc_train = train_df['timestamp'].value_counts()
ts_vc_test = test_df['timestamp'].value_counts()
f, ax = plt.subplots(figsize=(12, 6))
plt.bar(left=ts_vc_train.index, height=ts_vc_train)
plt.bar(left=ts_vc_test.index, height=ts_vc_test)
ax.xaxis.set_major_locator(years)
ax.xaxis.set_major_formatter(yearsFmt)
ax.set(title='Number of transactions by day', ylabel='count')
f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
sns.countplot(x=test_df['product_type'], ax=ax[0])
sns.countplot(x=train_df['product_type'], ax=ax[1])
ax[0].set(title='test', xlabel='product_type')
ax[1].set(title='train', xlabel='product_type')
f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
sns.countplot(x=test_df['state'], ax=ax[0])
sns.countplot(x=train_df['state'], ax=ax[1])
ax[0].set(title='test', xlabel='state')
ax[1].set(title='train', xlabel='state')
f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
sns.countplot(x=test_df['material'], ax=ax[0])
sns.countplot(x=train_df['material'], ax=ax[1])
ax[0].set(title='test', xlabel='material')
ax[1].set(title='train', xlabel='material')


# coding: utf-8

# This notebook is the reproduction of [A Very Extensive Sberbank Exploratory Analysis](https://www.kaggle.com/captcalculator/a-very-extensive-sberbank-exploratory-analysis) notebook in python. Here i copied only titles, for more detailed description please refer to original notebook.

# In[ ]:


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
color = sns.color_palette()

get_ipython().run_line_magic('matplotlib', 'inline')

pd.options.mode.chained_assignment = None  # default='warn'
pd.set_option('display.max_columns', 500)


# ## Training Data

# In[ ]:


train_df = pd.read_csv("../input/train.csv", parse_dates=['timestamp'])
train_df['price_doc_log'] = np.log1p(train_df['price_doc'])


# ## Missing Data

# In[ ]:


train_na = (train_df.isnull().sum() / len(train_df)) * 100
train_na = train_na.drop(train_na[train_na == 0].index).sort_values(ascending=False)


# In[ ]:


f, ax = plt.subplots(figsize=(12, 8))
plt.xticks(rotation='90')
sns.barplot(x=train_na.index, y=train_na)
ax.set(title='Percent missing data by feature', ylabel='% missing')


# ## Data Quality Issues

# In[ ]:


# state should be discrete valued between 1 and 4. There is a 33 in it that is cleary a data entry error
# Lets just replace it with the mode.
train_df.loc[train_df['state'] == 33, 'state'] = train_df['state'].mode().iloc[0]

# build_year has an erronus value 20052009. Since its unclear which it should be, let's replace with 2007
train_df.loc[train_df['build_year'] == 20052009, 'build_year'] = 2007


# ## Housing Internal Characteristics

# In[ ]:


internal_chars = ['full_sq', 'life_sq', 'floor', 'max_floor', 'build_year', 'num_room', 'kitch_sq', 'state', 'price_doc']
corrmat = train_df[internal_chars].corr()


# In[ ]:


f, ax = plt.subplots(figsize=(10, 7))
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=.5, annot=True)


# ## Area of Home and Number of Rooms

# In[ ]:


f, ax = plt.subplots(figsize=(10, 7))
plt.scatter(x=train_df['full_sq'], y=train_df['price_doc'], c='r')


# In[ ]:


f, ax = plt.subplots(figsize=(10, 7))
ind = train_df[train_df['full_sq'] > 2000].index
plt.scatter(x=train_df.drop(ind)['full_sq'], y=train_df.drop(ind)['price_doc'], c='r', alpha=0.5)
ax.set(title='Price by area in sq meters', xlabel='Area', ylabel='Price')


# In[ ]:


(train_df['life_sq'] > train_df['full_sq']).sum()


# In[ ]:


f, ax = plt.subplots(figsize=(10, 7))
sns.countplot(x=train_df['num_room'])
ax.set(title='Distribution of room count', xlabel='num_room')


# ## Sale Type

# In[ ]:


#can't plot density graph due to bug in kaggle kernels
#https://github.com/mwaskom/seaborn/issues/1103
#g = sns.FacetGrid(train_df, col="product_type", size=6)
#g.map(sns.kdeplot, "price_doc_log", color="r", shade=True)
#g.add_legend()
#ax.set(ylabel='density')

f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
train_df.drop(train_df['product_type'] == 'Investment')["price_doc_log"].plot.kde(ax=ax[0])
train_df.drop(train_df['product_type'] == 'OwnerOccupier')["price_doc_log"].plot.kde(ax=ax[1])
ax[0].set(xlabel='price_log')
ax[1].set(xlabel='price_log')


# In[ ]:


train_df.groupby('product_type')['price_doc'].median()


# ## Build Year

# In[ ]:


f, ax = plt.subplots(figsize=(12, 8))
plt.xticks(rotation='90')
ind = train_df[(train_df['build_year'] <= 1691) | (train_df['build_year'] >= 2018)].index
by_df = train_df.drop(ind).sort_values(by=['build_year'])
sns.countplot(x=by_df['build_year'])
ax.set(title='Distribution of build year')


# In[ ]:


f, ax = plt.subplots(figsize=(12, 6))
by_price = by_df.groupby('build_year')[['build_year', 'price_doc']].mean()
sns.regplot(x="build_year", y="price_doc", data=by_price, scatter=False, order=3, truncate=True)
plt.plot(by_price['build_year'], by_price['price_doc'], color='r')
ax.set(title='Mean price by year of build')


# ## Timestamp

# In[ ]:


f, ax = plt.subplots(figsize=(12, 6))
ts_df = train_df.groupby('timestamp')[['price_doc']].mean()
#sns.regplot(x="timestamp", y="price_doc", data=ts_df, scatter=False, truncate=True)
plt.plot(ts_df.index, ts_df['price_doc'], color='r', )
ax.set(title='Daily median price over time')


# In[ ]:


import datetime
import matplotlib.dates as mdates
years = mdates.YearLocator()   # every year
yearsFmt = mdates.DateFormatter('%Y')
ts_vc = train_df['timestamp'].value_counts()
f, ax = plt.subplots(figsize=(12, 6))
plt.bar(left=ts_vc.index, height=ts_vc)
ax.xaxis.set_major_locator(years)
ax.xaxis.set_major_formatter(yearsFmt)
ax.set(title='Sales volume over time', ylabel='Number of transactions')


# ## Home State/Material

# In[ ]:


f, ax = plt.subplots(figsize=(12, 8))
ts_df = train_df.groupby(by=[train_df.timestamp.dt.month])[['price_doc']].median()
plt.plot(ts_df.index, ts_df, color='r')
ax.set(title='Price by month of year')


# In[ ]:


f, ax = plt.subplots(figsize=(12, 8))
ind = train_df[train_df['state'].isnull()].index
train_df['price_doc_log10'] = np.log10(train_df['price_doc'])
sns.violinplot(x="state", y="price_doc_log10", data=train_df.drop(ind), inner="box")
# sns.swarmplot(x="state", y="price_doc_log10", data=train_df.dropna(), color="w", alpha=.2);
ax.set(title='Log10 of median price by state of home', xlabel='state', ylabel='log10(price)')


# In[ ]:


train_df.drop(ind).groupby('state')['price_doc'].mean()


# In[ ]:


f, ax = plt.subplots(figsize=(12, 8))
ind = train_df[train_df['material'].isnull()].index
sns.violinplot(x="material", y="price_doc_log", data=train_df.drop(ind), inner="box")
# sns.swarmplot(x="state", y="price_doc_log10", data=train_df.dropna(), color="w", alpha=.2);
ax.set(title='Distribution of price by build material', xlabel='material', ylabel='log(price)')


# In[ ]:


train_df.drop(ind).groupby('material')['price_doc'].median()


# ## Floor of Home

# In[ ]:


f, ax = plt.subplots(figsize=(12, 8))
plt.scatter(x=train_df['floor'], y=train_df['price_doc_log'], c='r', alpha=0.4)
sns.regplot(x="floor", y="price_doc_log", data=train_df, scatter=False, truncate=True)
ax.set(title='Price by floor of home', xlabel='floor', ylabel='log(price)')


# In[ ]:


f, ax = plt.subplots(figsize=(12, 8))
plt.scatter(x=train_df['max_floor'], y=train_df['price_doc_log'], c='r', alpha=0.4)
sns.regplot(x="max_floor", y="price_doc_log", data=train_df, scatter=False, truncate=True)
ax.set(title='Price by max floor of home', xlabel='max_floor', ylabel='log(price)')


# In[ ]:


f, ax = plt.subplots(figsize=(12, 8))
plt.scatter(x=train_df['floor'], y=train_df['max_floor'], c='r', alpha=0.4)
plt.plot([0, 80], [0, 80], color='.5')


# In[ ]:


train_df.loc[train_df['max_floor'] < train_df['floor'], ['id', 'floor','max_floor']].head(20)


# In[ ]:


## Demographic Characteristics


# In[ ]:


demo_vars = ['area_m', 'raion_popul', 'full_all', 'male_f', 'female_f', 'young_all', 'young_female', 
             'work_all', 'work_male', 'work_female', 'price_doc']
corrmat = train_df[demo_vars].corr()


# In[ ]:


f, ax = plt.subplots(figsize=(10, 7))
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=.5, annot=True)


# In[ ]:


train_df['sub_area'].unique().shape[0]


# In[ ]:


train_df['area_km'] = train_df['area_m'] / 1000000
train_df['density'] = train_df['raion_popul'] / train_df['area_km']
f, ax = plt.subplots(figsize=(10, 6))
sa_price = train_df.groupby('sub_area')[['density', 'price_doc']].median()
sns.regplot(x="density", y="price_doc", data=sa_price, scatter=True, truncate=True)
ax.set(title='Median home price by raion population density (people per sq. km)')


# In[ ]:


f, ax = plt.subplots(figsize=(10, 20))
sa_vc = train_df['sub_area'].value_counts()
sa_vc = pd.DataFrame({'sub_area':sa_vc.index, 'count': sa_vc.values})
ax = sns.barplot(x="count", y="sub_area", data=sa_vc, orient="h")
ax.set(title='Number of Transactions by District')
f.tight_layout()


# In[ ]:


train_df['work_share'] = train_df['work_all'] / train_df['raion_popul']
f, ax = plt.subplots(figsize=(12, 6))
sa_price = train_df.groupby('sub_area')[['work_share', 'price_doc']].mean()
sns.regplot(x="work_share", y="price_doc", data=sa_price, scatter=True, order=4, truncate=True)
ax.set(title='District mean home price by share of working age population')


# ## School Characteristics

# In[ ]:


school_chars = ['children_preschool', 'preschool_quota', 'preschool_education_centers_raion', 'children_school', 
                'school_quota', 'school_education_centers_raion', 'school_education_centers_top_20_raion', 
                'university_top_20_raion', 'additional_education_raion', 'additional_education_km', 'university_km', 'price_doc']
corrmat = train_df[school_chars].corr()


# In[ ]:


f, ax = plt.subplots(figsize=(10, 7))
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=.5, annot=True)


# In[ ]:


train_df['university_top_20_raion'].unique()


# In[ ]:


f, ax = plt.subplots(figsize=(12, 8))
sns.stripplot(x="university_top_20_raion", y="price_doc", data=train_df, jitter=True, alpha=.2, color=".8");
sns.boxplot(x="university_top_20_raion", y="price_doc", data=train_df)
ax.set(title='Distribution of home price by # of top universities in Raion', xlabel='university_top_20_raion', 
       ylabel='price_doc')


# ## Cultural/Recreational Characteristics

# In[ ]:


cult_chars = ['sport_objects_raion', 'culture_objects_top_25_raion', 'shopping_centers_raion', 'park_km', 'fitness_km', 
                'swim_pool_km', 'ice_rink_km','stadium_km', 'basketball_km', 'shopping_centers_km', 'big_church_km',
                'church_synagogue_km', 'mosque_km', 'theater_km', 'museum_km', 'exhibition_km', 'catering_km', 'price_doc']
corrmat = train_df[cult_chars].corr()


# In[ ]:


f, ax = plt.subplots(figsize=(12, 7))
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=.5, annot=True)


# In[ ]:


f, ax = plt.subplots(figsize=(10, 6))
so_price = train_df.groupby('sub_area')[['sport_objects_raion', 'price_doc']].median()
sns.regplot(x="sport_objects_raion", y="price_doc", data=so_price, scatter=True, truncate=True)
ax.set(title='Median Raion home price by # of sports objects in Raion')


# In[ ]:


f, ax = plt.subplots(figsize=(10, 6))
co_price = train_df.groupby('sub_area')[['culture_objects_top_25_raion', 'price_doc']].median()
sns.regplot(x="culture_objects_top_25_raion", y="price_doc", data=co_price, scatter=True, truncate=True)
ax.set(title='Median Raion home price by # of sports objects in Raion')


# In[ ]:


train_df.groupby('culture_objects_top_25')['price_doc'].median()


# In[ ]:


f, ax = plt.subplots(figsize=(10, 6))
sns.regplot(x="park_km", y="price_doc", data=train_df, scatter=True, truncate=True, scatter_kws={'color': 'r', 'alpha': .2})
ax.set(title='Median Raion home price by # of sports objects in Raion')


# ## Infrastructure Features

# In[ ]:


inf_features = ['nuclear_reactor_km', 'thermal_power_plant_km', 'power_transmission_line_km', 'incineration_km',
                'water_treatment_km', 'incineration_km', 'railroad_station_walk_km', 'railroad_station_walk_min', 
                'railroad_station_avto_km', 'railroad_station_avto_min', 'public_transport_station_km', 
                'public_transport_station_min_walk', 'water_km', 'mkad_km', 'ttk_km', 'sadovoe_km','bulvar_ring_km',
                'kremlin_km', 'price_doc']
corrmat = train_df[inf_features].corr()


# In[ ]:


f, ax = plt.subplots(figsize=(12, 7))
plt.xticks(rotation='90')
sns.heatmap(corrmat, square=True, linewidths=.5, annot=True)


# In[ ]:


f, ax = plt.subplots(figsize=(10, 6))
sns.regplot(x="kremlin_km", y="price_doc", data=train_df, scatter=True, truncate=True, scatter_kws={'color': 'r', 'alpha': .2})
ax.set(title='Home price by distance to Kremlin')


# ## Variable Importance

# In[ ]:


from sklearn.ensemble import RandomForestRegressor
from sklearn.preprocessing import LabelEncoder
X_train = train_df.drop(labels=['timestamp', 'id', 'incineration_raion'], axis=1).dropna()
y_train = X_train['price_doc']
X_train.drop('price_doc', axis=1, inplace=True)
for f in X_train.columns:
    if X_train[f].dtype == 'object':
        lbl = LabelEncoder()
        lbl.fit(X_train[f])
        X_train[f] = lbl.transform(X_train[f])
rf = RandomForestRegressor(random_state=0)
rf = rf.fit(X_train, y_train)


# In[ ]:


fi = list(zip(X_train.columns, rf.feature_importances_))
print('## rf variable importance')
d = [print('## %-40s%s' % (i)) for i in fi[:20]]


# ## Train vs Test Data

# In[ ]:


test_df = pd.read_csv("../input/test.csv", parse_dates=['timestamp'])
test_na = (test_df.isnull().sum() / len(test_df)) * 100
test_na = test_na.drop(test_na[test_na == 0].index).sort_values(ascending=False)


# In[ ]:


f, ax = plt.subplots(figsize=(12, 8))
plt.xticks(rotation='90')
sns.barplot(x=test_na.index, y=test_na)
ax.set(title='Percent missing data by feature', ylabel='% missing')


# In[ ]:


all_data = pd.concat([train_df.drop('price_doc', axis=1), test_df])
all_data['dataset'] = ''
l = len(train_df)
all_data.iloc[:l]['dataset'] = 'train'
all_data.iloc[l:]['dataset'] = 'test'
train_dataset = all_data['dataset'] == 'train'


# In[ ]:


f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
all_data['full_sq_log'] = np.log1p(all_data['full_sq'])
all_data.drop(train_dataset)["full_sq_log"].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)["full_sq_log"].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='full_sq_log')
ax[1].set(title='train', xlabel='full_sq_log')


# In[ ]:


f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
all_data['life_sq_log'] = np.log1p(all_data['life_sq'])
all_data.drop(train_dataset)["life_sq_log"].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)["life_sq_log"].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='life_sq_log')
ax[1].set(title='train', xlabel='life_sq_log')


# In[ ]:


f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
all_data['kitch_sq_log'] = np.log1p(all_data['kitch_sq'])
all_data.drop(train_dataset)["kitch_sq_log"].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)["kitch_sq_log"].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='kitch_sq_log')
ax[1].set(title='train', xlabel='kitch_sq_log')


# In[ ]:


f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
sns.countplot(x=test_df['num_room'], ax=ax[0])
sns.countplot(x=train_df['num_room'], ax=ax[1])
ax[0].set(title='test', xlabel='num_room')
ax[1].set(title='train', xlabel='num_room')


# In[ ]:


f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
all_data.drop(train_dataset)["floor"].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)["floor"].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='floor')
ax[1].set(title='train', xlabel='floor')


# In[ ]:


f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
all_data.drop(train_dataset)["max_floor"].plot.kde(ax=ax[0])
all_data.drop(~train_dataset)["max_floor"].plot.kde(ax=ax[1])
ax[0].set(title='test', xlabel='max_floor')
ax[1].set(title='train', xlabel='max_floor')


# In[ ]:


f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
ax[0].scatter(x=test_df['floor'], y=test_df['max_floor'], c='r', alpha=0.4)
ax[0].plot([0, 80], [0, 80], color='.5')
ax[1].scatter(x=train_df['floor'], y=train_df['max_floor'], c='r', alpha=0.4)
ax[1].plot([0, 80], [0, 80], color='.5')
ax[0].set(title='test', xlabel='floor', ylabel='max_floor')
ax[1].set(title='train', xlabel='floor', ylabel='max_floor')


# In[ ]:


years = mdates.YearLocator()   # every year
yearsFmt = mdates.DateFormatter('%Y')
ts_vc_train = train_df['timestamp'].value_counts()
ts_vc_test = test_df['timestamp'].value_counts()
f, ax = plt.subplots(figsize=(12, 6))
plt.bar(left=ts_vc_train.index, height=ts_vc_train)
plt.bar(left=ts_vc_test.index, height=ts_vc_test)
ax.xaxis.set_major_locator(years)
ax.xaxis.set_major_formatter(yearsFmt)
ax.set(title='Number of transactions by day', ylabel='count')


# In[ ]:


f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
sns.countplot(x=test_df['product_type'], ax=ax[0])
sns.countplot(x=train_df['product_type'], ax=ax[1])
ax[0].set(title='test', xlabel='product_type')
ax[1].set(title='train', xlabel='product_type')


# In[ ]:


f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
sns.countplot(x=test_df['state'], ax=ax[0])
sns.countplot(x=train_df['state'], ax=ax[1])
ax[0].set(title='test', xlabel='state')
ax[1].set(title='train', xlabel='state')


# In[ ]:


f, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 8), sharey=True)
sns.countplot(x=test_df['material'], ax=ax[0])
sns.countplot(x=train_df['material'], ax=ax[1])
ax[0].set(title='test', xlabel='material')
ax[1].set(title='train', xlabel='material')

