import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import os
import gc
#print("Data:\n",os.listdir("../input"))
import time
notebookstart= time.time()

# Models Packages
from sklearn import metrics
from sklearn.metrics import mean_squared_error
from sklearn import feature_selection
from sklearn.model_selection import train_test_split
from sklearn import preprocessing

# Gradient Boosting
import lightgbm as lgb
from sklearn.linear_model import Ridge
from sklearn.cross_validation import KFold

# Tf-Idf
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.pipeline import FeatureUnion
from scipy.sparse import hstack, csr_matrix
from nltk.corpus import stopwords

# Viz
import seaborn as sns
import matplotlib.pyplot as plt
import re
import string

NFOLDS = 5
SEED = 42
VALID = True
class SklearnWrapper(object):
    def __init__(self, clf, seed=0, params=None, seed_bool = True):
        if(seed_bool == True):
            params['random_state'] = seed
        self.clf = clf(**params)

    def train(self, x_train, y_train):
        self.clf.fit(x_train, y_train)

    def predict(self, x):
        return self.clf.predict(x)

def get_oof(clf, x_train, y, x_test):
    oof_train = np.zeros((ntrain,))
    oof_test = np.zeros((ntest,))
    oof_test_skf = np.empty((NFOLDS, ntest))

    for i, (train_index, test_index) in enumerate(kf):
        print('\nFold {}'.format(i))
        x_tr = x_train[train_index]
        y_tr = y[train_index]
        x_te = x_train[test_index]

        clf.train(x_tr, y_tr)

        oof_train[test_index] = clf.predict(x_te)
        oof_test_skf[i, :] = clf.predict(x_test)

    oof_test[:] = oof_test_skf.mean(axis=0)
    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)

def cleanName(text):
    try:
        textProc = text.lower()
        # textProc = " ".join(map(str.strip, re.split('(\d+)',textProc)))
        #regex = re.compile(u'[^[:alpha:]]')
        #textProc = regex.sub(" ", textProc)
        textProc = re.sub('[!@#$_“”¨«»®´·º½¾¿¡§£₤‘’]', '', textProc)
        textProc = " ".join(textProc.split())
        return textProc
    except:
        return "name error"


def rmse(y, y0):
    assert len(y) == len(y0)
    return np.sqrt(np.mean(np.power((y - y0), 2)))

print("\nData Load Stage")
training = pd.read_csv('../input/train.csv', index_col = "item_id", parse_dates = ["activation_date"])
traindex = training.index
testing = pd.read_csv('../input/test.csv', index_col = "item_id", parse_dates = ["activation_date"])
testdex = testing.index
#testcode #turn on this if you want to test the code only
#training = training[0:100]
#testing = testing[0:50]

ntrain = training.shape[0]
ntest = testing.shape[0]

kf = KFold(ntrain, n_folds=NFOLDS, shuffle=True, random_state=SEED)

y = training.deal_probability.copy()
training.drop("deal_probability",axis=1, inplace=True)
print('Train shape: {} Rows, {} Columns'.format(*training.shape))
print('Test shape: {} Rows, {} Columns'.format(*testing.shape))

print("Combine Train and Test")
df = pd.concat([training,testing],axis=0)
del training, testing
gc.collect()
print('\nAll Data shape: {} Rows, {} Columns'.format(*df.shape))


print("Feature Engineering")
df["price"] = np.log(df["price"]+0.001)
df["price"].fillna(df.price.mean(),inplace=True)
df["image_top_1"].fillna(-999,inplace=True)

print("\nCreate Time Variables")
df["Weekday"] = df['activation_date'].dt.weekday
df["Weekd of Year"] = df['activation_date'].dt.week
df["Day of Month"] = df['activation_date'].dt.day

# Create Validation Index and Remove Dead Variables
training_index = df.loc[df.activation_date<=pd.to_datetime('2017-04-07')].index
validation_index = df.loc[df.activation_date>=pd.to_datetime('2017-04-08')].index
df.drop(["activation_date","image"],axis=1,inplace=True)

print("\nEncode Variables")
categorical = ["user_id","region","city","parent_category_name","category_name","user_type","image_top_1","param_1","param_2","param_3"]
print("Encoding :",categorical)

# Encoder:
lbl = preprocessing.LabelEncoder()
for col in categorical:
    df[col].fillna('Unknown')
    df[col] = lbl.fit_transform(df[col].astype(str))

print("\nText Features")

# Feature Engineering

# Meta Text Features
textfeats = ["description", "title"]
df['desc_punc'] = df['description'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]))

df['title'] = df['title'].apply(lambda x: cleanName(x))
df["description"]   = df["description"].apply(lambda x: cleanName(x))
'''
skewness = skewness[abs(skewness) > 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))

from scipy.special import boxcox1p
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
    #all_data[feat] += 1
    all_data[feat] = boxcox1p(all_data[feat], lam)
'''
for cols in textfeats:
    df[cols] = df[cols].astype(str)
    df[cols] = df[cols].astype(str).fillna('missing') # FILL NA
    df[cols] = df[cols].str.lower() # Lowercase all text, so that capitalized words dont get treated differently
    df[cols + '_num_words'] = df[cols].apply(lambda comment: len(comment.split())) # Count number of Words
    df[cols + '_num_unique_words'] = df[cols].apply(lambda comment: len(set(w for w in comment.split())))
    df[cols + '_words_vs_unique'] = df[cols+'_num_unique_words'] / df[cols+'_num_words'] * 100 # Count Unique Words

#df = pd.concat([training,testing],axis=0)
cols = df.columns[:]
for col in cols:
    df[col].fillna(-999, inplace=True)

#df['weekday_price'] = df["price"] / (df["Weekday"] + 1) *100
#df['weekday_image_top'] = df["image_top_1"] / (df["Weekday"] + 1) * 100
#df['city_price'] = df["price"] / (df["city"] + 1) * 100
#df['regin_image_top'] = df["image_top_1"] / (df["region"] + 1) * 100

#df['weekday_image_top'] = df["image_top_1"] / (df["Weekday"] + 1) * 100
#df['user_cate'] = df["user_type"] / (df["category_name"] + 1) * 100
#df['regin_image_top'] = df["image_top_1"] /( df["region"] + 1) * 100

# not sure about this method to
#interval = (-1, 1, 2, 3,4,5, 6,7, 8,9,10,11,12,13, 14,15,16,17,18,19,20,99)
#cats = ['0-1', '1-2', '2-3', '3-4', '4-5','5-6', '6-7', '7-8', '8-9', '9-10','10-11','11-12','12-13','13-14','14-15','15-16','16-17','17-18','18-19','19-20','20-99']
#df["price_log_divide_weekday_num"] = pd.cut(df["price"] / (df["Weekday"] + 1), interval, labels=cats)
#df["price_log_divide_city_num"] = pd.cut(df["price"] / (df["city"] + 1), interval, labels=cats)
#df["price_log_divide_image_top_num"] = pd.cut(df["price"] / (df["image_top_1"] + 1), interval, labels=cats)


print("\n[TF-IDF] Term Frequency Inverse Document Frequency Stage")
russian_stop = set(stopwords.words('russian'))

tfidf_para = {
    "stop_words": russian_stop,
    "analyzer": 'word',
    "token_pattern": r'\w{1,}',
    "sublinear_tf": True,
    "dtype": np.float32,
    "norm": 'l2',
    #"min_df":5,
    #"max_df":.9,
    "smooth_idf":False
}


def get_col(col_name): return lambda x: x[col_name]
##I added to the max_features of the description. It did not change my score much but it may be worth investigating
vectorizer = FeatureUnion([
        ('description',TfidfVectorizer(
            ngram_range=(1, 2),
            max_features=17000,
            **tfidf_para,
            preprocessor=get_col('description'))),
        ('title',CountVectorizer(
            ngram_range=(1, 2),
            stop_words = russian_stop,
            #max_features=7000,
            preprocessor=get_col('title')))
    ])

start_vect=time.time()

#Fit my vectorizer on the entire dataset instead of the training rows
#Score improved by .0001
vectorizer.fit(df.to_dict('records'))

ready_df = vectorizer.transform(df.to_dict('records'))
tfvocab = vectorizer.get_feature_names()
print("Vectorization Runtime: %0.2f Minutes"%((time.time() - start_vect)/60))

# Drop Text Cols
textfeats = ["description", "title"]
df.drop(textfeats, axis=1,inplace=True)

from sklearn.metrics import mean_squared_error
from math import sqrt

ridge_params = {'alpha':30.0, 'fit_intercept':True, 'normalize':False, 'copy_X':True,
                'max_iter':None, 'tol':0.001, 'solver':'auto', 'random_state':SEED}

#Ridge oof method from Faron's kernel
#I was using this to analyze my vectorization, but figured it would be interesting to add the results back into the dataset
#It doesn't really add much to the score, but it does help lightgbm converge faster
ridge = SklearnWrapper(clf=Ridge, seed = SEED, params = ridge_params)
ridge_oof_train, ridge_oof_test = get_oof(ridge, ready_df[:ntrain], y, ready_df[ntrain:])

rms = sqrt(mean_squared_error(y, ridge_oof_train))
print('Ridge OOF RMSE: {}'.format(rms))

print("Modeling Stage")

ridge_preds = np.concatenate([ridge_oof_train, ridge_oof_test])

df['ridge_preds'] = ridge_preds

# Combine Dense Features with Sparse Text Bag of Words Features
X = hstack([csr_matrix(df.loc[traindex,:].values),ready_df[0:traindex.shape[0]]]) # Sparse Matrix
testing = hstack([csr_matrix(df.loc[testdex,:].values),ready_df[traindex.shape[0]:]])
tfvocab = df.columns.tolist() + tfvocab
for shape in [X,testing]:
    print("{} Rows and {} Cols".format(*shape.shape))
print("Feature Names Length: ",len(tfvocab))
del df
gc.collect();

print("\nModeling Stage")

del ridge_preds,vectorizer,ready_df
gc.collect();



from sklearn.linear_model import ElasticNet, Lasso,  BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor,  GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
#from sklearn.model_selection import KFold, cross_val_score, train_test_split
#from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb
from mlxtend.regressor import StackingRegressor
from mlxtend.data import boston_housing_data
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Ridge
from sklearn.svm import SVR

from xgboost import XGBRegressor
from lightgbm import LGBMRegressor

from sklearn.svm import SVR
from sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor, AdaBoostRegressor
from sklearn.linear_model import LinearRegression
from sklearn.tree import DecisionTreeRegressor
"""
lr = LinearRegression()
svr_lin = SVR(kernel='linear')
ridge = Ridge(random_state=1)
svr_rbf = SVR(kernel='rbf')

stregr = StackingRegressor(regressors=[svr_lin, lr, ridge],
                           meta_regressor=svr_rbf)

# Training the stacking classifier

stregr.fit(X, y)
stregr.predict(X)
"""

# lgb params
lgb_params = {}
#lgb_params['n_estimators'] = 50
lgb_params['max_bin'] = 8
lgb_params['learning_rate'] = 0.019 # shrinkage_rate
lgb_params['metric'] = 'l1'          # or 'mae'
lgb_params['sub_feature'] = 0.35
lgb_params['bagging_fraction'] = 0.85 # sub_row
lgb_params['bagging_freq'] = 40
lgb_params['num_leaves'] = 512        # num_leaf
lgb_params['min_data'] = 500         # min_data_in_leaf
lgb_params['min_hessian'] = 0.05     # min_sum_hessian_in_leaf
lgb_params['verbose'] = 0
lgb_params['feature_fraction_seed'] = 2
lgb_params['bagging_seed'] = 3


"""

print("Light Gradient Boosting Regressor")
lgbm_params =  {
    'task': 'train',
    'boosting_type': 'gbdt',
    'objective': 'regression',
    'metric': 'rmse',
    # 'max_depth': 15,
    'num_leaves': 270,
    'feature_fraction': 0.5,
    'bagging_fraction': 0.75,
    'bagging_freq': 2,
    'learning_rate': 0.016,
    'verbose': 0
}
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
                              learning_rate=0.05, n_estimators=720,
                              max_bin = 55, bagging_fraction = 0.8,
                              bagging_freq = 5, feature_fraction = 0.2319,
                              feature_fraction_seed=9, bagging_seed=9,
                              min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)

model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
                             learning_rate=0.05, max_depth=3,
                             min_child_weight=1.7817, n_estimators=2200,
                             reg_alpha=0.4640, reg_lambda=0.8571,
                             subsample=0.5213, silent=1,
                             random_state =7, nthread = -1)
lasso = make_pipeline(RobustScaler(), Lasso(alpha =0.0005, random_state=1))
ENet = make_pipeline(RobustScaler(), ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
                                   max_depth=4, max_features='sqrt',
                                   min_samples_leaf=15, min_samples_split=10,
                                   loss='huber', random_state =5)

"""
# rf params
rf_params = {}
rf_params['n_estimators'] = 50
rf_params['max_depth'] = 8
rf_params['min_samples_split'] = 100
rf_params['min_samples_leaf'] = 30

# xgb params
xgb_params = {}
#xgb_params['n_estimators'] = 50
xgb_params['min_child_weight'] = 12
xgb_params['learning_rate'] = 0.37
xgb_params['max_depth'] = 6
xgb_params['subsample'] = 0.77
xgb_params['reg_lambda'] = 0.8
xgb_params['reg_alpha'] = 0.4
xgb_params['base_score'] = 0
#xgb_params['seed'] = 400
xgb_params['silent'] = 1

# XGB model
xgb_model = XGBRegressor(**xgb_params)

# lgb model
lgb_model = LGBMRegressor(**lgb_params)

# RF model
rf_model = RandomForestRegressor(**rf_params)

# ET model
et_model = ExtraTreesRegressor()

# SVR model
# SVM is too slow in more then 10000 set
#svr_model = SVR(kernel='rbf', C=1.0, epsilon=0.05)

# DecsionTree model
dt_model = DecisionTreeRegressor()

# AdaBoost model
ada_model = AdaBoostRegressor()



#y_test = stack.fit_predict(x_train, y_train, x_test)
#stack = Ensemble(n_splits=5,
#        stacker=LinearRegression(),
#        base_models=(rf_model, xgb_model, lgb_model, et_model, ada_model))

stregr = StackingRegressor(regressors=[rf_model, et_model],
                           meta_regressor=LinearRegression())


if VALID == False:
    X_train, X_valid, y_train, y_valid = train_test_split(
        X, y, test_size=0.10, random_state=23)

    # LGBM Dataset Formatting
    sttrain = stregr.fit(X_train, y_train
                    #feature_name=tfvocab,
                    #categorical_feature = categorical
                    )
    lgvalid = stregr.fit(X_valid, y_valid
                    #feature_name=tfvocab,
                    #categorical_feature = categorical
                    )
    del X, X_train; gc.collect()

    # Go Go Go
    stregr_clf = stregr.predict(
        #lgbm_params,
        sttrain
        #num_boost_round=20000,
        #valid_sets=[sttrain, lgvalid],
        #valid_names=['train','valid'],
        #early_stopping_rounds=50,
        #verbose_eval=100
    )
    print("Model Evaluation Stage")
    print('RMSE:', np.sqrt(metrics.mean_squared_error(y_valid, stregr_clf.predict(X_valid))))
    del X_valid ; gc.collect()
#stregr.fit(X, y)
#stregr.predict(X)
else:
    # LGBM Dataset Formatting
    sttrain = stregr.fit(X, y
                    #feature_name=tfvocab,
                    #categorical_feature = categorical
                    )
    del X; gc.collect()
    # Go Go Go
    stregr_clf = stregr.predict(
        #lgbm_params,
        sttrain
        #num_boost_round=1550,
        #verbose_eval=100
    )



# Feature Importance Plot
#f, ax = plt.subplots(figsize=[7,10])
#stregr.plot_importance(stregr_clf, max_num_features=50, ax=ax)
#plt.title("Light GBM Feature Importance")
#plt.savefig('feature_import.png')

print("Model Evaluation Stage")
lgpred = stregr_clf.predict(testing)

#Mixing lightgbm with ridge. I haven't really tested if this improves the score or not
#blend = 0.95*lgpred + 0.05*ridge_oof_test[:,0]
lgsub = pd.DataFrame(lgpred,columns=["deal_probability"],index=testdex)
lgsub['deal_probability'].clip(0.0, 1.0, inplace=True) # Between 0 and 1
lgsub.to_csv("lgsub.csv",index=True,header=True)
#print("Model Runtime: %0.2f Minutes"%((time.time() - modelstart)/60))
print("Notebook Runtime: %0.2f Minutes"%((time.time() - notebookstart)/60))

f249fdeade72d4be5f9daaf91e92200b53ff335c
<mask token>


class SklearnWrapper(object):

    def __init__(self, clf, seed=0, params=None, seed_bool=True):
        if seed_bool == True:
            params['random_state'] = seed
        self.clf = clf(**params)

    def train(self, x_train, y_train):
        self.clf.fit(x_train, y_train)

    def predict(self, x):
        return self.clf.predict(x)


<mask token>


def cleanName(text):
    try:
        textProc = text.lower()
        textProc = re.sub('[!@#$_“”¨«»®´·º½¾¿¡§£₤‘’]', '', textProc)
        textProc = ' '.join(textProc.split())
        return textProc
    except:
        return 'name error'


def rmse(y, y0):
    assert len(y) == len(y0)
    return np.sqrt(np.mean(np.power(y - y0, 2)))


<mask token>


def get_col(col_name):
    return lambda x: x[col_name]


<mask token>

<mask token>


class SklearnWrapper(object):

    def __init__(self, clf, seed=0, params=None, seed_bool=True):
        if seed_bool == True:
            params['random_state'] = seed
        self.clf = clf(**params)

    def train(self, x_train, y_train):
        self.clf.fit(x_train, y_train)

    def predict(self, x):
        return self.clf.predict(x)


def get_oof(clf, x_train, y, x_test):
    oof_train = np.zeros((ntrain,))
    oof_test = np.zeros((ntest,))
    oof_test_skf = np.empty((NFOLDS, ntest))
    for i, (train_index, test_index) in enumerate(kf):
        print('\nFold {}'.format(i))
        x_tr = x_train[train_index]
        y_tr = y[train_index]
        x_te = x_train[test_index]
        clf.train(x_tr, y_tr)
        oof_train[test_index] = clf.predict(x_te)
        oof_test_skf[i, :] = clf.predict(x_test)
    oof_test[:] = oof_test_skf.mean(axis=0)
    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)


def cleanName(text):
    try:
        textProc = text.lower()
        textProc = re.sub('[!@#$_“”¨«»®´·º½¾¿¡§£₤‘’]', '', textProc)
        textProc = ' '.join(textProc.split())
        return textProc
    except:
        return 'name error'


def rmse(y, y0):
    assert len(y) == len(y0)
    return np.sqrt(np.mean(np.power(y - y0, 2)))


print('\nData Load Stage')
<mask token>
training.drop('deal_probability', axis=1, inplace=True)
print('Train shape: {} Rows, {} Columns'.format(*training.shape))
print('Test shape: {} Rows, {} Columns'.format(*testing.shape))
print('Combine Train and Test')
<mask token>
del training, testing
gc.collect()
print("""
All Data shape: {} Rows, {} Columns""".format(*df.shape))
print('Feature Engineering')
<mask token>
df['price'].fillna(df.price.mean(), inplace=True)
df['image_top_1'].fillna(-999, inplace=True)
print("""
Create Time Variables""")
<mask token>
df.drop(['activation_date', 'image'], axis=1, inplace=True)
print("""
Encode Variables""")
<mask token>
print('Encoding :', categorical)
<mask token>
for col in categorical:
    df[col].fillna('Unknown')
    df[col] = lbl.fit_transform(df[col].astype(str))
print('\nText Features')
<mask token>
for cols in textfeats:
    df[cols] = df[cols].astype(str)
    df[cols] = df[cols].astype(str).fillna('missing')
    df[cols] = df[cols].str.lower()
    df[cols + '_num_words'] = df[cols].apply(lambda comment: len(comment.
        split()))
    df[cols + '_num_unique_words'] = df[cols].apply(lambda comment: len(set
        (w for w in comment.split())))
    df[cols + '_words_vs_unique'] = df[cols + '_num_unique_words'] / df[
        cols + '_num_words'] * 100
<mask token>
for col in cols:
    df[col].fillna(-999, inplace=True)
print("""
[TF-IDF] Term Frequency Inverse Document Frequency Stage""")
<mask token>


def get_col(col_name):
    return lambda x: x[col_name]


<mask token>
vectorizer.fit(df.to_dict('records'))
<mask token>
print('Vectorization Runtime: %0.2f Minutes' % ((time.time() - start_vect) /
    60))
<mask token>
df.drop(textfeats, axis=1, inplace=True)
<mask token>
print('Ridge OOF RMSE: {}'.format(rms))
print('Modeling Stage')
<mask token>
for shape in [X, testing]:
    print('{} Rows and {} Cols'.format(*shape.shape))
print('Feature Names Length: ', len(tfvocab))
del df
gc.collect()
print('\nModeling Stage')
del ridge_preds, vectorizer, ready_df
gc.collect()
<mask token>
if VALID == False:
    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=
        0.1, random_state=23)
    sttrain = stregr.fit(X_train, y_train)
    lgvalid = stregr.fit(X_valid, y_valid)
    del X, X_train
    gc.collect()
    stregr_clf = stregr.predict(sttrain)
    print('Model Evaluation Stage')
    print('RMSE:', np.sqrt(metrics.mean_squared_error(y_valid, stregr_clf.
        predict(X_valid))))
    del X_valid
    gc.collect()
else:
    sttrain = stregr.fit(X, y)
    del X
    gc.collect()
    stregr_clf = stregr.predict(sttrain)
print('Model Evaluation Stage')
<mask token>
lgsub['deal_probability'].clip(0.0, 1.0, inplace=True)
lgsub.to_csv('lgsub.csv', index=True, header=True)
print('Notebook Runtime: %0.2f Minutes' % ((time.time() - notebookstart) / 60))

<mask token>
notebookstart = time.time()
<mask token>
NFOLDS = 5
SEED = 42
VALID = True


class SklearnWrapper(object):

    def __init__(self, clf, seed=0, params=None, seed_bool=True):
        if seed_bool == True:
            params['random_state'] = seed
        self.clf = clf(**params)

    def train(self, x_train, y_train):
        self.clf.fit(x_train, y_train)

    def predict(self, x):
        return self.clf.predict(x)


def get_oof(clf, x_train, y, x_test):
    oof_train = np.zeros((ntrain,))
    oof_test = np.zeros((ntest,))
    oof_test_skf = np.empty((NFOLDS, ntest))
    for i, (train_index, test_index) in enumerate(kf):
        print('\nFold {}'.format(i))
        x_tr = x_train[train_index]
        y_tr = y[train_index]
        x_te = x_train[test_index]
        clf.train(x_tr, y_tr)
        oof_train[test_index] = clf.predict(x_te)
        oof_test_skf[i, :] = clf.predict(x_test)
    oof_test[:] = oof_test_skf.mean(axis=0)
    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)


def cleanName(text):
    try:
        textProc = text.lower()
        textProc = re.sub('[!@#$_“”¨«»®´·º½¾¿¡§£₤‘’]', '', textProc)
        textProc = ' '.join(textProc.split())
        return textProc
    except:
        return 'name error'


def rmse(y, y0):
    assert len(y) == len(y0)
    return np.sqrt(np.mean(np.power(y - y0, 2)))


print('\nData Load Stage')
training = pd.read_csv('../input/train.csv', index_col='item_id',
    parse_dates=['activation_date'])
traindex = training.index
testing = pd.read_csv('../input/test.csv', index_col='item_id', parse_dates
    =['activation_date'])
testdex = testing.index
ntrain = training.shape[0]
ntest = testing.shape[0]
kf = KFold(ntrain, n_folds=NFOLDS, shuffle=True, random_state=SEED)
y = training.deal_probability.copy()
training.drop('deal_probability', axis=1, inplace=True)
print('Train shape: {} Rows, {} Columns'.format(*training.shape))
print('Test shape: {} Rows, {} Columns'.format(*testing.shape))
print('Combine Train and Test')
df = pd.concat([training, testing], axis=0)
del training, testing
gc.collect()
print("""
All Data shape: {} Rows, {} Columns""".format(*df.shape))
print('Feature Engineering')
df['price'] = np.log(df['price'] + 0.001)
df['price'].fillna(df.price.mean(), inplace=True)
df['image_top_1'].fillna(-999, inplace=True)
print("""
Create Time Variables""")
df['Weekday'] = df['activation_date'].dt.weekday
df['Weekd of Year'] = df['activation_date'].dt.week
df['Day of Month'] = df['activation_date'].dt.day
training_index = df.loc[df.activation_date <= pd.to_datetime('2017-04-07')
    ].index
validation_index = df.loc[df.activation_date >= pd.to_datetime('2017-04-08')
    ].index
df.drop(['activation_date', 'image'], axis=1, inplace=True)
print("""
Encode Variables""")
categorical = ['user_id', 'region', 'city', 'parent_category_name',
    'category_name', 'user_type', 'image_top_1', 'param_1', 'param_2',
    'param_3']
print('Encoding :', categorical)
lbl = preprocessing.LabelEncoder()
for col in categorical:
    df[col].fillna('Unknown')
    df[col] = lbl.fit_transform(df[col].astype(str))
print('\nText Features')
textfeats = ['description', 'title']
df['desc_punc'] = df['description'].apply(lambda x: len([c for c in str(x) if
    c in string.punctuation]))
df['title'] = df['title'].apply(lambda x: cleanName(x))
df['description'] = df['description'].apply(lambda x: cleanName(x))
<mask token>
for cols in textfeats:
    df[cols] = df[cols].astype(str)
    df[cols] = df[cols].astype(str).fillna('missing')
    df[cols] = df[cols].str.lower()
    df[cols + '_num_words'] = df[cols].apply(lambda comment: len(comment.
        split()))
    df[cols + '_num_unique_words'] = df[cols].apply(lambda comment: len(set
        (w for w in comment.split())))
    df[cols + '_words_vs_unique'] = df[cols + '_num_unique_words'] / df[
        cols + '_num_words'] * 100
cols = df.columns[:]
for col in cols:
    df[col].fillna(-999, inplace=True)
print("""
[TF-IDF] Term Frequency Inverse Document Frequency Stage""")
russian_stop = set(stopwords.words('russian'))
tfidf_para = {'stop_words': russian_stop, 'analyzer': 'word',
    'token_pattern': '\\w{1,}', 'sublinear_tf': True, 'dtype': np.float32,
    'norm': 'l2', 'smooth_idf': False}


def get_col(col_name):
    return lambda x: x[col_name]


vectorizer = FeatureUnion([('description', TfidfVectorizer(ngram_range=(1, 
    2), max_features=17000, **tfidf_para, preprocessor=get_col(
    'description'))), ('title', CountVectorizer(ngram_range=(1, 2),
    stop_words=russian_stop, preprocessor=get_col('title')))])
start_vect = time.time()
vectorizer.fit(df.to_dict('records'))
ready_df = vectorizer.transform(df.to_dict('records'))
tfvocab = vectorizer.get_feature_names()
print('Vectorization Runtime: %0.2f Minutes' % ((time.time() - start_vect) /
    60))
textfeats = ['description', 'title']
df.drop(textfeats, axis=1, inplace=True)
<mask token>
ridge_params = {'alpha': 30.0, 'fit_intercept': True, 'normalize': False,
    'copy_X': True, 'max_iter': None, 'tol': 0.001, 'solver': 'auto',
    'random_state': SEED}
ridge = SklearnWrapper(clf=Ridge, seed=SEED, params=ridge_params)
ridge_oof_train, ridge_oof_test = get_oof(ridge, ready_df[:ntrain], y,
    ready_df[ntrain:])
rms = sqrt(mean_squared_error(y, ridge_oof_train))
print('Ridge OOF RMSE: {}'.format(rms))
print('Modeling Stage')
ridge_preds = np.concatenate([ridge_oof_train, ridge_oof_test])
df['ridge_preds'] = ridge_preds
X = hstack([csr_matrix(df.loc[traindex, :].values), ready_df[0:traindex.
    shape[0]]])
testing = hstack([csr_matrix(df.loc[testdex, :].values), ready_df[traindex.
    shape[0]:]])
tfvocab = df.columns.tolist() + tfvocab
for shape in [X, testing]:
    print('{} Rows and {} Cols'.format(*shape.shape))
print('Feature Names Length: ', len(tfvocab))
del df
gc.collect()
print('\nModeling Stage')
del ridge_preds, vectorizer, ready_df
gc.collect()
<mask token>
lgb_params = {}
lgb_params['max_bin'] = 8
lgb_params['learning_rate'] = 0.019
lgb_params['metric'] = 'l1'
lgb_params['sub_feature'] = 0.35
lgb_params['bagging_fraction'] = 0.85
lgb_params['bagging_freq'] = 40
lgb_params['num_leaves'] = 512
lgb_params['min_data'] = 500
lgb_params['min_hessian'] = 0.05
lgb_params['verbose'] = 0
lgb_params['feature_fraction_seed'] = 2
lgb_params['bagging_seed'] = 3
<mask token>
rf_params = {}
rf_params['n_estimators'] = 50
rf_params['max_depth'] = 8
rf_params['min_samples_split'] = 100
rf_params['min_samples_leaf'] = 30
xgb_params = {}
xgb_params['min_child_weight'] = 12
xgb_params['learning_rate'] = 0.37
xgb_params['max_depth'] = 6
xgb_params['subsample'] = 0.77
xgb_params['reg_lambda'] = 0.8
xgb_params['reg_alpha'] = 0.4
xgb_params['base_score'] = 0
xgb_params['silent'] = 1
xgb_model = XGBRegressor(**xgb_params)
lgb_model = LGBMRegressor(**lgb_params)
rf_model = RandomForestRegressor(**rf_params)
et_model = ExtraTreesRegressor()
dt_model = DecisionTreeRegressor()
ada_model = AdaBoostRegressor()
stregr = StackingRegressor(regressors=[rf_model, et_model], meta_regressor=
    LinearRegression())
if VALID == False:
    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=
        0.1, random_state=23)
    sttrain = stregr.fit(X_train, y_train)
    lgvalid = stregr.fit(X_valid, y_valid)
    del X, X_train
    gc.collect()
    stregr_clf = stregr.predict(sttrain)
    print('Model Evaluation Stage')
    print('RMSE:', np.sqrt(metrics.mean_squared_error(y_valid, stregr_clf.
        predict(X_valid))))
    del X_valid
    gc.collect()
else:
    sttrain = stregr.fit(X, y)
    del X
    gc.collect()
    stregr_clf = stregr.predict(sttrain)
print('Model Evaluation Stage')
lgpred = stregr_clf.predict(testing)
lgsub = pd.DataFrame(lgpred, columns=['deal_probability'], index=testdex)
lgsub['deal_probability'].clip(0.0, 1.0, inplace=True)
lgsub.to_csv('lgsub.csv', index=True, header=True)
print('Notebook Runtime: %0.2f Minutes' % ((time.time() - notebookstart) / 60))

import numpy as np
import pandas as pd
import os
import gc
import time
notebookstart = time.time()
from sklearn import metrics
from sklearn.metrics import mean_squared_error
from sklearn import feature_selection
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
import lightgbm as lgb
from sklearn.linear_model import Ridge
from sklearn.cross_validation import KFold
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.pipeline import FeatureUnion
from scipy.sparse import hstack, csr_matrix
from nltk.corpus import stopwords
import seaborn as sns
import matplotlib.pyplot as plt
import re
import string
NFOLDS = 5
SEED = 42
VALID = True


class SklearnWrapper(object):

    def __init__(self, clf, seed=0, params=None, seed_bool=True):
        if seed_bool == True:
            params['random_state'] = seed
        self.clf = clf(**params)

    def train(self, x_train, y_train):
        self.clf.fit(x_train, y_train)

    def predict(self, x):
        return self.clf.predict(x)


def get_oof(clf, x_train, y, x_test):
    oof_train = np.zeros((ntrain,))
    oof_test = np.zeros((ntest,))
    oof_test_skf = np.empty((NFOLDS, ntest))
    for i, (train_index, test_index) in enumerate(kf):
        print('\nFold {}'.format(i))
        x_tr = x_train[train_index]
        y_tr = y[train_index]
        x_te = x_train[test_index]
        clf.train(x_tr, y_tr)
        oof_train[test_index] = clf.predict(x_te)
        oof_test_skf[i, :] = clf.predict(x_test)
    oof_test[:] = oof_test_skf.mean(axis=0)
    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)


def cleanName(text):
    try:
        textProc = text.lower()
        textProc = re.sub('[!@#$_“”¨«»®´·º½¾¿¡§£₤‘’]', '', textProc)
        textProc = ' '.join(textProc.split())
        return textProc
    except:
        return 'name error'


def rmse(y, y0):
    assert len(y) == len(y0)
    return np.sqrt(np.mean(np.power(y - y0, 2)))


print('\nData Load Stage')
training = pd.read_csv('../input/train.csv', index_col='item_id',
    parse_dates=['activation_date'])
traindex = training.index
testing = pd.read_csv('../input/test.csv', index_col='item_id', parse_dates
    =['activation_date'])
testdex = testing.index
ntrain = training.shape[0]
ntest = testing.shape[0]
kf = KFold(ntrain, n_folds=NFOLDS, shuffle=True, random_state=SEED)
y = training.deal_probability.copy()
training.drop('deal_probability', axis=1, inplace=True)
print('Train shape: {} Rows, {} Columns'.format(*training.shape))
print('Test shape: {} Rows, {} Columns'.format(*testing.shape))
print('Combine Train and Test')
df = pd.concat([training, testing], axis=0)
del training, testing
gc.collect()
print("""
All Data shape: {} Rows, {} Columns""".format(*df.shape))
print('Feature Engineering')
df['price'] = np.log(df['price'] + 0.001)
df['price'].fillna(df.price.mean(), inplace=True)
df['image_top_1'].fillna(-999, inplace=True)
print("""
Create Time Variables""")
df['Weekday'] = df['activation_date'].dt.weekday
df['Weekd of Year'] = df['activation_date'].dt.week
df['Day of Month'] = df['activation_date'].dt.day
training_index = df.loc[df.activation_date <= pd.to_datetime('2017-04-07')
    ].index
validation_index = df.loc[df.activation_date >= pd.to_datetime('2017-04-08')
    ].index
df.drop(['activation_date', 'image'], axis=1, inplace=True)
print("""
Encode Variables""")
categorical = ['user_id', 'region', 'city', 'parent_category_name',
    'category_name', 'user_type', 'image_top_1', 'param_1', 'param_2',
    'param_3']
print('Encoding :', categorical)
lbl = preprocessing.LabelEncoder()
for col in categorical:
    df[col].fillna('Unknown')
    df[col] = lbl.fit_transform(df[col].astype(str))
print('\nText Features')
textfeats = ['description', 'title']
df['desc_punc'] = df['description'].apply(lambda x: len([c for c in str(x) if
    c in string.punctuation]))
df['title'] = df['title'].apply(lambda x: cleanName(x))
df['description'] = df['description'].apply(lambda x: cleanName(x))
<mask token>
for cols in textfeats:
    df[cols] = df[cols].astype(str)
    df[cols] = df[cols].astype(str).fillna('missing')
    df[cols] = df[cols].str.lower()
    df[cols + '_num_words'] = df[cols].apply(lambda comment: len(comment.
        split()))
    df[cols + '_num_unique_words'] = df[cols].apply(lambda comment: len(set
        (w for w in comment.split())))
    df[cols + '_words_vs_unique'] = df[cols + '_num_unique_words'] / df[
        cols + '_num_words'] * 100
cols = df.columns[:]
for col in cols:
    df[col].fillna(-999, inplace=True)
print("""
[TF-IDF] Term Frequency Inverse Document Frequency Stage""")
russian_stop = set(stopwords.words('russian'))
tfidf_para = {'stop_words': russian_stop, 'analyzer': 'word',
    'token_pattern': '\\w{1,}', 'sublinear_tf': True, 'dtype': np.float32,
    'norm': 'l2', 'smooth_idf': False}


def get_col(col_name):
    return lambda x: x[col_name]


vectorizer = FeatureUnion([('description', TfidfVectorizer(ngram_range=(1, 
    2), max_features=17000, **tfidf_para, preprocessor=get_col(
    'description'))), ('title', CountVectorizer(ngram_range=(1, 2),
    stop_words=russian_stop, preprocessor=get_col('title')))])
start_vect = time.time()
vectorizer.fit(df.to_dict('records'))
ready_df = vectorizer.transform(df.to_dict('records'))
tfvocab = vectorizer.get_feature_names()
print('Vectorization Runtime: %0.2f Minutes' % ((time.time() - start_vect) /
    60))
textfeats = ['description', 'title']
df.drop(textfeats, axis=1, inplace=True)
from sklearn.metrics import mean_squared_error
from math import sqrt
ridge_params = {'alpha': 30.0, 'fit_intercept': True, 'normalize': False,
    'copy_X': True, 'max_iter': None, 'tol': 0.001, 'solver': 'auto',
    'random_state': SEED}
ridge = SklearnWrapper(clf=Ridge, seed=SEED, params=ridge_params)
ridge_oof_train, ridge_oof_test = get_oof(ridge, ready_df[:ntrain], y,
    ready_df[ntrain:])
rms = sqrt(mean_squared_error(y, ridge_oof_train))
print('Ridge OOF RMSE: {}'.format(rms))
print('Modeling Stage')
ridge_preds = np.concatenate([ridge_oof_train, ridge_oof_test])
df['ridge_preds'] = ridge_preds
X = hstack([csr_matrix(df.loc[traindex, :].values), ready_df[0:traindex.
    shape[0]]])
testing = hstack([csr_matrix(df.loc[testdex, :].values), ready_df[traindex.
    shape[0]:]])
tfvocab = df.columns.tolist() + tfvocab
for shape in [X, testing]:
    print('{} Rows and {} Cols'.format(*shape.shape))
print('Feature Names Length: ', len(tfvocab))
del df
gc.collect()
print('\nModeling Stage')
del ridge_preds, vectorizer, ready_df
gc.collect()
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
import xgboost as xgb
import lightgbm as lgb
from mlxtend.regressor import StackingRegressor
from mlxtend.data import boston_housing_data
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Ridge
from sklearn.svm import SVR
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from sklearn.svm import SVR
from sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor, AdaBoostRegressor
from sklearn.linear_model import LinearRegression
from sklearn.tree import DecisionTreeRegressor
<mask token>
lgb_params = {}
lgb_params['max_bin'] = 8
lgb_params['learning_rate'] = 0.019
lgb_params['metric'] = 'l1'
lgb_params['sub_feature'] = 0.35
lgb_params['bagging_fraction'] = 0.85
lgb_params['bagging_freq'] = 40
lgb_params['num_leaves'] = 512
lgb_params['min_data'] = 500
lgb_params['min_hessian'] = 0.05
lgb_params['verbose'] = 0
lgb_params['feature_fraction_seed'] = 2
lgb_params['bagging_seed'] = 3
<mask token>
rf_params = {}
rf_params['n_estimators'] = 50
rf_params['max_depth'] = 8
rf_params['min_samples_split'] = 100
rf_params['min_samples_leaf'] = 30
xgb_params = {}
xgb_params['min_child_weight'] = 12
xgb_params['learning_rate'] = 0.37
xgb_params['max_depth'] = 6
xgb_params['subsample'] = 0.77
xgb_params['reg_lambda'] = 0.8
xgb_params['reg_alpha'] = 0.4
xgb_params['base_score'] = 0
xgb_params['silent'] = 1
xgb_model = XGBRegressor(**xgb_params)
lgb_model = LGBMRegressor(**lgb_params)
rf_model = RandomForestRegressor(**rf_params)
et_model = ExtraTreesRegressor()
dt_model = DecisionTreeRegressor()
ada_model = AdaBoostRegressor()
stregr = StackingRegressor(regressors=[rf_model, et_model], meta_regressor=
    LinearRegression())
if VALID == False:
    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=
        0.1, random_state=23)
    sttrain = stregr.fit(X_train, y_train)
    lgvalid = stregr.fit(X_valid, y_valid)
    del X, X_train
    gc.collect()
    stregr_clf = stregr.predict(sttrain)
    print('Model Evaluation Stage')
    print('RMSE:', np.sqrt(metrics.mean_squared_error(y_valid, stregr_clf.
        predict(X_valid))))
    del X_valid
    gc.collect()
else:
    sttrain = stregr.fit(X, y)
    del X
    gc.collect()
    stregr_clf = stregr.predict(sttrain)
print('Model Evaluation Stage')
lgpred = stregr_clf.predict(testing)
lgsub = pd.DataFrame(lgpred, columns=['deal_probability'], index=testdex)
lgsub['deal_probability'].clip(0.0, 1.0, inplace=True)
lgsub.to_csv('lgsub.csv', index=True, header=True)
print('Notebook Runtime: %0.2f Minutes' % ((time.time() - notebookstart) / 60))

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import os
import gc
#print("Data:\n",os.listdir("../input"))
import time
notebookstart= time.time()

# Models Packages
from sklearn import metrics
from sklearn.metrics import mean_squared_error
from sklearn import feature_selection
from sklearn.model_selection import train_test_split
from sklearn import preprocessing

# Gradient Boosting
import lightgbm as lgb
from sklearn.linear_model import Ridge
from sklearn.cross_validation import KFold

# Tf-Idf
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.pipeline import FeatureUnion
from scipy.sparse import hstack, csr_matrix
from nltk.corpus import stopwords

# Viz
import seaborn as sns
import matplotlib.pyplot as plt
import re
import string

NFOLDS = 5
SEED = 42
VALID = True
class SklearnWrapper(object):
    def __init__(self, clf, seed=0, params=None, seed_bool = True):
        if(seed_bool == True):
            params['random_state'] = seed
        self.clf = clf(**params)

    def train(self, x_train, y_train):
        self.clf.fit(x_train, y_train)

    def predict(self, x):
        return self.clf.predict(x)

def get_oof(clf, x_train, y, x_test):
    oof_train = np.zeros((ntrain,))
    oof_test = np.zeros((ntest,))
    oof_test_skf = np.empty((NFOLDS, ntest))

    for i, (train_index, test_index) in enumerate(kf):
        print('\nFold {}'.format(i))
        x_tr = x_train[train_index]
        y_tr = y[train_index]
        x_te = x_train[test_index]

        clf.train(x_tr, y_tr)

        oof_train[test_index] = clf.predict(x_te)
        oof_test_skf[i, :] = clf.predict(x_test)

    oof_test[:] = oof_test_skf.mean(axis=0)
    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)

def cleanName(text):
    try:
        textProc = text.lower()
        # textProc = " ".join(map(str.strip, re.split('(\d+)',textProc)))
        #regex = re.compile(u'[^[:alpha:]]')
        #textProc = regex.sub(" ", textProc)
        textProc = re.sub('[!@#$_“”¨«»®´·º½¾¿¡§£₤‘’]', '', textProc)
        textProc = " ".join(textProc.split())
        return textProc
    except:
        return "name error"


def rmse(y, y0):
    assert len(y) == len(y0)
    return np.sqrt(np.mean(np.power((y - y0), 2)))

print("\nData Load Stage")
training = pd.read_csv('../input/train.csv', index_col = "item_id", parse_dates = ["activation_date"])
traindex = training.index
testing = pd.read_csv('../input/test.csv', index_col = "item_id", parse_dates = ["activation_date"])
testdex = testing.index
#testcode #turn on this if you want to test the code only
#training = training[0:100]
#testing = testing[0:50]

ntrain = training.shape[0]
ntest = testing.shape[0]

kf = KFold(ntrain, n_folds=NFOLDS, shuffle=True, random_state=SEED)

y = training.deal_probability.copy()
training.drop("deal_probability",axis=1, inplace=True)
print('Train shape: {} Rows, {} Columns'.format(*training.shape))
print('Test shape: {} Rows, {} Columns'.format(*testing.shape))

print("Combine Train and Test")
df = pd.concat([training,testing],axis=0)
del training, testing
gc.collect()
print('\nAll Data shape: {} Rows, {} Columns'.format(*df.shape))


print("Feature Engineering")
df["price"] = np.log(df["price"]+0.001)
df["price"].fillna(df.price.mean(),inplace=True)
df["image_top_1"].fillna(-999,inplace=True)

print("\nCreate Time Variables")
df["Weekday"] = df['activation_date'].dt.weekday
df["Weekd of Year"] = df['activation_date'].dt.week
df["Day of Month"] = df['activation_date'].dt.day

# Create Validation Index and Remove Dead Variables
training_index = df.loc[df.activation_date<=pd.to_datetime('2017-04-07')].index
validation_index = df.loc[df.activation_date>=pd.to_datetime('2017-04-08')].index
df.drop(["activation_date","image"],axis=1,inplace=True)

print("\nEncode Variables")
categorical = ["user_id","region","city","parent_category_name","category_name","user_type","image_top_1","param_1","param_2","param_3"]
print("Encoding :",categorical)

# Encoder:
lbl = preprocessing.LabelEncoder()
for col in categorical:
    df[col].fillna('Unknown')
    df[col] = lbl.fit_transform(df[col].astype(str))

print("\nText Features")

# Feature Engineering

# Meta Text Features
textfeats = ["description", "title"]
df['desc_punc'] = df['description'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]))

df['title'] = df['title'].apply(lambda x: cleanName(x))
df["description"]   = df["description"].apply(lambda x: cleanName(x))
'''
skewness = skewness[abs(skewness) > 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))

from scipy.special import boxcox1p
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
    #all_data[feat] += 1
    all_data[feat] = boxcox1p(all_data[feat], lam)
'''
for cols in textfeats:
    df[cols] = df[cols].astype(str)
    df[cols] = df[cols].astype(str).fillna('missing') # FILL NA
    df[cols] = df[cols].str.lower() # Lowercase all text, so that capitalized words dont get treated differently
    df[cols + '_num_words'] = df[cols].apply(lambda comment: len(comment.split())) # Count number of Words
    df[cols + '_num_unique_words'] = df[cols].apply(lambda comment: len(set(w for w in comment.split())))
    df[cols + '_words_vs_unique'] = df[cols+'_num_unique_words'] / df[cols+'_num_words'] * 100 # Count Unique Words

#df = pd.concat([training,testing],axis=0)
cols = df.columns[:]
for col in cols:
    df[col].fillna(-999, inplace=True)

#df['weekday_price'] = df["price"] / (df["Weekday"] + 1) *100
#df['weekday_image_top'] = df["image_top_1"] / (df["Weekday"] + 1) * 100
#df['city_price'] = df["price"] / (df["city"] + 1) * 100
#df['regin_image_top'] = df["image_top_1"] / (df["region"] + 1) * 100

#df['weekday_image_top'] = df["image_top_1"] / (df["Weekday"] + 1) * 100
#df['user_cate'] = df["user_type"] / (df["category_name"] + 1) * 100
#df['regin_image_top'] = df["image_top_1"] /( df["region"] + 1) * 100

# not sure about this method to
#interval = (-1, 1, 2, 3,4,5, 6,7, 8,9,10,11,12,13, 14,15,16,17,18,19,20,99)
#cats = ['0-1', '1-2', '2-3', '3-4', '4-5','5-6', '6-7', '7-8', '8-9', '9-10','10-11','11-12','12-13','13-14','14-15','15-16','16-17','17-18','18-19','19-20','20-99']
#df["price_log_divide_weekday_num"] = pd.cut(df["price"] / (df["Weekday"] + 1), interval, labels=cats)
#df["price_log_divide_city_num"] = pd.cut(df["price"] / (df["city"] + 1), interval, labels=cats)
#df["price_log_divide_image_top_num"] = pd.cut(df["price"] / (df["image_top_1"] + 1), interval, labels=cats)


print("\n[TF-IDF] Term Frequency Inverse Document Frequency Stage")
russian_stop = set(stopwords.words('russian'))

tfidf_para = {
    "stop_words": russian_stop,
    "analyzer": 'word',
    "token_pattern": r'\w{1,}',
    "sublinear_tf": True,
    "dtype": np.float32,
    "norm": 'l2',
    #"min_df":5,
    #"max_df":.9,
    "smooth_idf":False
}


def get_col(col_name): return lambda x: x[col_name]
##I added to the max_features of the description. It did not change my score much but it may be worth investigating
vectorizer = FeatureUnion([
        ('description',TfidfVectorizer(
            ngram_range=(1, 2),
            max_features=17000,
            **tfidf_para,
            preprocessor=get_col('description'))),
        ('title',CountVectorizer(
            ngram_range=(1, 2),
            stop_words = russian_stop,
            #max_features=7000,
            preprocessor=get_col('title')))
    ])

start_vect=time.time()

#Fit my vectorizer on the entire dataset instead of the training rows
#Score improved by .0001
vectorizer.fit(df.to_dict('records'))

ready_df = vectorizer.transform(df.to_dict('records'))
tfvocab = vectorizer.get_feature_names()
print("Vectorization Runtime: %0.2f Minutes"%((time.time() - start_vect)/60))

# Drop Text Cols
textfeats = ["description", "title"]
df.drop(textfeats, axis=1,inplace=True)

from sklearn.metrics import mean_squared_error
from math import sqrt

ridge_params = {'alpha':30.0, 'fit_intercept':True, 'normalize':False, 'copy_X':True,
                'max_iter':None, 'tol':0.001, 'solver':'auto', 'random_state':SEED}

#Ridge oof method from Faron's kernel
#I was using this to analyze my vectorization, but figured it would be interesting to add the results back into the dataset
#It doesn't really add much to the score, but it does help lightgbm converge faster
ridge = SklearnWrapper(clf=Ridge, seed = SEED, params = ridge_params)
ridge_oof_train, ridge_oof_test = get_oof(ridge, ready_df[:ntrain], y, ready_df[ntrain:])

rms = sqrt(mean_squared_error(y, ridge_oof_train))
print('Ridge OOF RMSE: {}'.format(rms))

print("Modeling Stage")

ridge_preds = np.concatenate([ridge_oof_train, ridge_oof_test])

df['ridge_preds'] = ridge_preds

# Combine Dense Features with Sparse Text Bag of Words Features
X = hstack([csr_matrix(df.loc[traindex,:].values),ready_df[0:traindex.shape[0]]]) # Sparse Matrix
testing = hstack([csr_matrix(df.loc[testdex,:].values),ready_df[traindex.shape[0]:]])
tfvocab = df.columns.tolist() + tfvocab
for shape in [X,testing]:
    print("{} Rows and {} Cols".format(*shape.shape))
print("Feature Names Length: ",len(tfvocab))
del df
gc.collect();

print("\nModeling Stage")

del ridge_preds,vectorizer,ready_df
gc.collect();



from sklearn.linear_model import ElasticNet, Lasso,  BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor,  GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
#from sklearn.model_selection import KFold, cross_val_score, train_test_split
#from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb
from mlxtend.regressor import StackingRegressor
from mlxtend.data import boston_housing_data
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Ridge
from sklearn.svm import SVR

from xgboost import XGBRegressor
from lightgbm import LGBMRegressor

from sklearn.svm import SVR
from sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor, AdaBoostRegressor
from sklearn.linear_model import LinearRegression
from sklearn.tree import DecisionTreeRegressor
"""
lr = LinearRegression()
svr_lin = SVR(kernel='linear')
ridge = Ridge(random_state=1)
svr_rbf = SVR(kernel='rbf')

stregr = StackingRegressor(regressors=[svr_lin, lr, ridge],
                           meta_regressor=svr_rbf)

# Training the stacking classifier

stregr.fit(X, y)
stregr.predict(X)
"""

# lgb params
lgb_params = {}
#lgb_params['n_estimators'] = 50
lgb_params['max_bin'] = 8
lgb_params['learning_rate'] = 0.019 # shrinkage_rate
lgb_params['metric'] = 'l1'          # or 'mae'
lgb_params['sub_feature'] = 0.35
lgb_params['bagging_fraction'] = 0.85 # sub_row
lgb_params['bagging_freq'] = 40
lgb_params['num_leaves'] = 512        # num_leaf
lgb_params['min_data'] = 500         # min_data_in_leaf
lgb_params['min_hessian'] = 0.05     # min_sum_hessian_in_leaf
lgb_params['verbose'] = 0
lgb_params['feature_fraction_seed'] = 2
lgb_params['bagging_seed'] = 3


"""

print("Light Gradient Boosting Regressor")
lgbm_params =  {
    'task': 'train',
    'boosting_type': 'gbdt',
    'objective': 'regression',
    'metric': 'rmse',
    # 'max_depth': 15,
    'num_leaves': 270,
    'feature_fraction': 0.5,
    'bagging_fraction': 0.75,
    'bagging_freq': 2,
    'learning_rate': 0.016,
    'verbose': 0
}
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
                              learning_rate=0.05, n_estimators=720,
                              max_bin = 55, bagging_fraction = 0.8,
                              bagging_freq = 5, feature_fraction = 0.2319,
                              feature_fraction_seed=9, bagging_seed=9,
                              min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)

model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
                             learning_rate=0.05, max_depth=3,
                             min_child_weight=1.7817, n_estimators=2200,
                             reg_alpha=0.4640, reg_lambda=0.8571,
                             subsample=0.5213, silent=1,
                             random_state =7, nthread = -1)
lasso = make_pipeline(RobustScaler(), Lasso(alpha =0.0005, random_state=1))
ENet = make_pipeline(RobustScaler(), ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
                                   max_depth=4, max_features='sqrt',
                                   min_samples_leaf=15, min_samples_split=10,
                                   loss='huber', random_state =5)

"""
# rf params
rf_params = {}
rf_params['n_estimators'] = 50
rf_params['max_depth'] = 8
rf_params['min_samples_split'] = 100
rf_params['min_samples_leaf'] = 30

# xgb params
xgb_params = {}
#xgb_params['n_estimators'] = 50
xgb_params['min_child_weight'] = 12
xgb_params['learning_rate'] = 0.37
xgb_params['max_depth'] = 6
xgb_params['subsample'] = 0.77
xgb_params['reg_lambda'] = 0.8
xgb_params['reg_alpha'] = 0.4
xgb_params['base_score'] = 0
#xgb_params['seed'] = 400
xgb_params['silent'] = 1

# XGB model
xgb_model = XGBRegressor(**xgb_params)

# lgb model
lgb_model = LGBMRegressor(**lgb_params)

# RF model
rf_model = RandomForestRegressor(**rf_params)

# ET model
et_model = ExtraTreesRegressor()

# SVR model
# SVM is too slow in more then 10000 set
#svr_model = SVR(kernel='rbf', C=1.0, epsilon=0.05)

# DecsionTree model
dt_model = DecisionTreeRegressor()

# AdaBoost model
ada_model = AdaBoostRegressor()



#y_test = stack.fit_predict(x_train, y_train, x_test)
#stack = Ensemble(n_splits=5,
#        stacker=LinearRegression(),
#        base_models=(rf_model, xgb_model, lgb_model, et_model, ada_model))

stregr = StackingRegressor(regressors=[rf_model, et_model],
                           meta_regressor=LinearRegression())


if VALID == False:
    X_train, X_valid, y_train, y_valid = train_test_split(
        X, y, test_size=0.10, random_state=23)

    # LGBM Dataset Formatting
    sttrain = stregr.fit(X_train, y_train
                    #feature_name=tfvocab,
                    #categorical_feature = categorical
                    )
    lgvalid = stregr.fit(X_valid, y_valid
                    #feature_name=tfvocab,
                    #categorical_feature = categorical
                    )
    del X, X_train; gc.collect()

    # Go Go Go
    stregr_clf = stregr.predict(
        #lgbm_params,
        sttrain
        #num_boost_round=20000,
        #valid_sets=[sttrain, lgvalid],
        #valid_names=['train','valid'],
        #early_stopping_rounds=50,
        #verbose_eval=100
    )
    print("Model Evaluation Stage")
    print('RMSE:', np.sqrt(metrics.mean_squared_error(y_valid, stregr_clf.predict(X_valid))))
    del X_valid ; gc.collect()
#stregr.fit(X, y)
#stregr.predict(X)
else:
    # LGBM Dataset Formatting
    sttrain = stregr.fit(X, y
                    #feature_name=tfvocab,
                    #categorical_feature = categorical
                    )
    del X; gc.collect()
    # Go Go Go
    stregr_clf = stregr.predict(
        #lgbm_params,
        sttrain
        #num_boost_round=1550,
        #verbose_eval=100
    )



# Feature Importance Plot
#f, ax = plt.subplots(figsize=[7,10])
#stregr.plot_importance(stregr_clf, max_num_features=50, ax=ax)
#plt.title("Light GBM Feature Importance")
#plt.savefig('feature_import.png')

print("Model Evaluation Stage")
lgpred = stregr_clf.predict(testing)

#Mixing lightgbm with ridge. I haven't really tested if this improves the score or not
#blend = 0.95*lgpred + 0.05*ridge_oof_test[:,0]
lgsub = pd.DataFrame(lgpred,columns=["deal_probability"],index=testdex)
lgsub['deal_probability'].clip(0.0, 1.0, inplace=True) # Between 0 and 1
lgsub.to_csv("lgsub.csv",index=True,header=True)
#print("Model Runtime: %0.2f Minutes"%((time.time() - modelstart)/60))
print("Notebook Runtime: %0.2f Minutes"%((time.time() - notebookstart)/60))
