"""
LGBM Regression on TfIDF of text features and One-Hot-Encoded Categoricals
Featues based on Alexandu Papiu's (https://www.kaggle.com/apapiu) script: https://www.kaggle.com/apapiu/ridge-script
LGBM based on InfiniteWing's (https://www.kaggle.com/infinitewing) script: https://www.kaggle.com/infinitewing/lightgbm-example
"""

import pandas as pd
import numpy as np
import scipy

from sklearn.linear_model import Ridge, LogisticRegression
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.preprocessing import LabelBinarizer
import lightgbm as lgb
import time

import gc

start = fixstart = time.time()

NUM_BRANDS = 2500
NAME_MIN_DF = 10
MAX_FEAT_DESCP = 50000

print("Reading in Data")

df_train = pd.read_csv('../input/train.tsv', sep='\t')
df_test = pd.read_csv('../input/test.tsv', sep='\t')

df = pd.concat([df_train, df_test], 0)
nrow_train = df_train.shape[0]
y_train = np.log1p(df_train["price"])

del df_train
gc.collect()

print(df.memory_usage(deep = True))

df["category_name"] = df["category_name"].fillna("Other").astype("category")
df["brand_name"] = df["brand_name"].fillna("unknown")

pop_brands = df["brand_name"].value_counts().index[:NUM_BRANDS]
df.loc[~df["brand_name"].isin(pop_brands), "brand_name"] = "Other"

df["item_description"] = df["item_description"].fillna("None")
df["item_condition_id"] = df["item_condition_id"].astype("category")
df["brand_name"] = df["brand_name"].astype("category")

# description contains interesting words
interesting_words = ['new', 'perfect', 'fit', 'used', 'super', 'cute', 'excellent',
                     'great', 'retail', '[rm]', 'never' ]
for word in interesting_words:
    df[word] = df['item_description'].apply(lambda x : word in x.lower())


print(df.memory_usage(deep = True))

print("Encodings")
count = CountVectorizer(min_df=NAME_MIN_DF)
X_name = count.fit_transform(df["name"])

print("Category Encoders")
unique_categories = pd.Series("/".join(df["category_name"].unique().astype("str")).split("/")).unique()
count_category = CountVectorizer()
X_category = count_category.fit_transform(df["category_name"])

print("Descp encoders")
count_descp = TfidfVectorizer(max_features = MAX_FEAT_DESCP, 
                              ngram_range = (1,3),
                              stop_words = "english")
X_descp = count_descp.fit_transform(df["item_description"])

print("Brand encoders")
vect_brand = LabelBinarizer(sparse_output=True)
X_brand = vect_brand.fit_transform(df["brand_name"])

print("Dummy Encoders")
X_dummies = scipy.sparse.csr_matrix(pd.get_dummies(df[[
    "item_condition_id", "shipping"]], sparse = True).values)

X = scipy.sparse.hstack((X_dummies, 
                         X_descp,
                         X_brand,
                         X_category,
                         X_name)).tocsr()

print([X_dummies.shape, X_category.shape, 
       X_name.shape, X_descp.shape, X_brand.shape])

X_train = X[:nrow_train]
X_test = X[nrow_train:]

end = time.time()
print("Time taken reading and encoding  {}.".format((end-start)))

#LGBM Training
start = time.time()

params = {
    'learning_rate': 0.8,
    'application': 'regression',
    'max_depth': 3,
    'num_leaves': 100,
    'verbosity': -1,
    'metric': 'RMSE', 
    'max_bin':8192
}


train_X, valid_X, train_y, valid_y = train_test_split(X_train, y_train, 
                                                      test_size = 0.1, 
                                                      random_state = 144) 
d_train = lgb.Dataset(train_X, label=train_y)
d_valid = lgb.Dataset(valid_X, label=valid_y)
watchlist = [d_train, d_valid]

model = lgb.train(params, train_set=d_train, num_boost_round=3000, 
                  valid_sets=watchlist, 
                  early_stopping_rounds=50, verbose_eval=100) 
preds = model.predict(X_test)
end = time.time()
print("Time taken training LGBM  {}.".format((end-start)))

#Ridge Training
start = time.time()

modelr = Ridge(solver = "lsqr", fit_intercept=False)
modelr.fit(X_train, y_train)

preds += modelr.predict(X_test)
preds /= 2
end = time.time()
print("Time taken training Ridge  {}.".format((end-start)))


df_test["price"] = np.expm1(preds)
df_test[["test_id", "price"]].to_csv("ff_LGBM_Ridge_3.csv", index = False)
end = time.time()
print("Time taken by the Kernel  {}.".format((end-fixstart)))

2d23cc52c9bfa5c3dab20262d2037cb343c25cf5
<mask token>

<mask token>
print('Reading in Data')
<mask token>
del df_train
gc.collect()
print(df.memory_usage(deep=True))
<mask token>
for word in interesting_words:
    df[word] = df['item_description'].apply(lambda x: word in x.lower())
print(df.memory_usage(deep=True))
print('Encodings')
<mask token>
print('Category Encoders')
<mask token>
print('Descp encoders')
<mask token>
print('Brand encoders')
<mask token>
print('Dummy Encoders')
<mask token>
print([X_dummies.shape, X_category.shape, X_name.shape, X_descp.shape,
    X_brand.shape])
<mask token>
print('Time taken reading and encoding  {}.'.format(end - start))
<mask token>
print('Time taken training LGBM  {}.'.format(end - start))
<mask token>
modelr.fit(X_train, y_train)
preds += modelr.predict(X_test)
preds /= 2
<mask token>
print('Time taken training Ridge  {}.'.format(end - start))
<mask token>
df_test[['test_id', 'price']].to_csv('ff_LGBM_Ridge_3.csv', index=False)
<mask token>
print('Time taken by the Kernel  {}.'.format(end - fixstart))

<mask token>
start = fixstart = time.time()
NUM_BRANDS = 2500
NAME_MIN_DF = 10
MAX_FEAT_DESCP = 50000
print('Reading in Data')
df_train = pd.read_csv('../input/train.tsv', sep='\t')
df_test = pd.read_csv('../input/test.tsv', sep='\t')
df = pd.concat([df_train, df_test], 0)
nrow_train = df_train.shape[0]
y_train = np.log1p(df_train['price'])
del df_train
gc.collect()
print(df.memory_usage(deep=True))
df['category_name'] = df['category_name'].fillna('Other').astype('category')
df['brand_name'] = df['brand_name'].fillna('unknown')
pop_brands = df['brand_name'].value_counts().index[:NUM_BRANDS]
df.loc[~df['brand_name'].isin(pop_brands), 'brand_name'] = 'Other'
df['item_description'] = df['item_description'].fillna('None')
df['item_condition_id'] = df['item_condition_id'].astype('category')
df['brand_name'] = df['brand_name'].astype('category')
interesting_words = ['new', 'perfect', 'fit', 'used', 'super', 'cute',
    'excellent', 'great', 'retail', '[rm]', 'never']
for word in interesting_words:
    df[word] = df['item_description'].apply(lambda x: word in x.lower())
print(df.memory_usage(deep=True))
print('Encodings')
count = CountVectorizer(min_df=NAME_MIN_DF)
X_name = count.fit_transform(df['name'])
print('Category Encoders')
unique_categories = pd.Series('/'.join(df['category_name'].unique().astype(
    'str')).split('/')).unique()
count_category = CountVectorizer()
X_category = count_category.fit_transform(df['category_name'])
print('Descp encoders')
count_descp = TfidfVectorizer(max_features=MAX_FEAT_DESCP, ngram_range=(1, 
    3), stop_words='english')
X_descp = count_descp.fit_transform(df['item_description'])
print('Brand encoders')
vect_brand = LabelBinarizer(sparse_output=True)
X_brand = vect_brand.fit_transform(df['brand_name'])
print('Dummy Encoders')
X_dummies = scipy.sparse.csr_matrix(pd.get_dummies(df[['item_condition_id',
    'shipping']], sparse=True).values)
X = scipy.sparse.hstack((X_dummies, X_descp, X_brand, X_category, X_name)
    ).tocsr()
print([X_dummies.shape, X_category.shape, X_name.shape, X_descp.shape,
    X_brand.shape])
X_train = X[:nrow_train]
X_test = X[nrow_train:]
end = time.time()
print('Time taken reading and encoding  {}.'.format(end - start))
start = time.time()
params = {'learning_rate': 0.8, 'application': 'regression', 'max_depth': 3,
    'num_leaves': 100, 'verbosity': -1, 'metric': 'RMSE', 'max_bin': 8192}
train_X, valid_X, train_y, valid_y = train_test_split(X_train, y_train,
    test_size=0.1, random_state=144)
d_train = lgb.Dataset(train_X, label=train_y)
d_valid = lgb.Dataset(valid_X, label=valid_y)
watchlist = [d_train, d_valid]
model = lgb.train(params, train_set=d_train, num_boost_round=3000,
    valid_sets=watchlist, early_stopping_rounds=50, verbose_eval=100)
preds = model.predict(X_test)
end = time.time()
print('Time taken training LGBM  {}.'.format(end - start))
start = time.time()
modelr = Ridge(solver='lsqr', fit_intercept=False)
modelr.fit(X_train, y_train)
preds += modelr.predict(X_test)
preds /= 2
end = time.time()
print('Time taken training Ridge  {}.'.format(end - start))
df_test['price'] = np.expm1(preds)
df_test[['test_id', 'price']].to_csv('ff_LGBM_Ridge_3.csv', index=False)
end = time.time()
print('Time taken by the Kernel  {}.'.format(end - fixstart))

<mask token>
import pandas as pd
import numpy as np
import scipy
from sklearn.linear_model import Ridge, LogisticRegression
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.preprocessing import LabelBinarizer
import lightgbm as lgb
import time
import gc
start = fixstart = time.time()
NUM_BRANDS = 2500
NAME_MIN_DF = 10
MAX_FEAT_DESCP = 50000
print('Reading in Data')
df_train = pd.read_csv('../input/train.tsv', sep='\t')
df_test = pd.read_csv('../input/test.tsv', sep='\t')
df = pd.concat([df_train, df_test], 0)
nrow_train = df_train.shape[0]
y_train = np.log1p(df_train['price'])
del df_train
gc.collect()
print(df.memory_usage(deep=True))
df['category_name'] = df['category_name'].fillna('Other').astype('category')
df['brand_name'] = df['brand_name'].fillna('unknown')
pop_brands = df['brand_name'].value_counts().index[:NUM_BRANDS]
df.loc[~df['brand_name'].isin(pop_brands), 'brand_name'] = 'Other'
df['item_description'] = df['item_description'].fillna('None')
df['item_condition_id'] = df['item_condition_id'].astype('category')
df['brand_name'] = df['brand_name'].astype('category')
interesting_words = ['new', 'perfect', 'fit', 'used', 'super', 'cute',
    'excellent', 'great', 'retail', '[rm]', 'never']
for word in interesting_words:
    df[word] = df['item_description'].apply(lambda x: word in x.lower())
print(df.memory_usage(deep=True))
print('Encodings')
count = CountVectorizer(min_df=NAME_MIN_DF)
X_name = count.fit_transform(df['name'])
print('Category Encoders')
unique_categories = pd.Series('/'.join(df['category_name'].unique().astype(
    'str')).split('/')).unique()
count_category = CountVectorizer()
X_category = count_category.fit_transform(df['category_name'])
print('Descp encoders')
count_descp = TfidfVectorizer(max_features=MAX_FEAT_DESCP, ngram_range=(1, 
    3), stop_words='english')
X_descp = count_descp.fit_transform(df['item_description'])
print('Brand encoders')
vect_brand = LabelBinarizer(sparse_output=True)
X_brand = vect_brand.fit_transform(df['brand_name'])
print('Dummy Encoders')
X_dummies = scipy.sparse.csr_matrix(pd.get_dummies(df[['item_condition_id',
    'shipping']], sparse=True).values)
X = scipy.sparse.hstack((X_dummies, X_descp, X_brand, X_category, X_name)
    ).tocsr()
print([X_dummies.shape, X_category.shape, X_name.shape, X_descp.shape,
    X_brand.shape])
X_train = X[:nrow_train]
X_test = X[nrow_train:]
end = time.time()
print('Time taken reading and encoding  {}.'.format(end - start))
start = time.time()
params = {'learning_rate': 0.8, 'application': 'regression', 'max_depth': 3,
    'num_leaves': 100, 'verbosity': -1, 'metric': 'RMSE', 'max_bin': 8192}
train_X, valid_X, train_y, valid_y = train_test_split(X_train, y_train,
    test_size=0.1, random_state=144)
d_train = lgb.Dataset(train_X, label=train_y)
d_valid = lgb.Dataset(valid_X, label=valid_y)
watchlist = [d_train, d_valid]
model = lgb.train(params, train_set=d_train, num_boost_round=3000,
    valid_sets=watchlist, early_stopping_rounds=50, verbose_eval=100)
preds = model.predict(X_test)
end = time.time()
print('Time taken training LGBM  {}.'.format(end - start))
start = time.time()
modelr = Ridge(solver='lsqr', fit_intercept=False)
modelr.fit(X_train, y_train)
preds += modelr.predict(X_test)
preds /= 2
end = time.time()
print('Time taken training Ridge  {}.'.format(end - start))
df_test['price'] = np.expm1(preds)
df_test[['test_id', 'price']].to_csv('ff_LGBM_Ridge_3.csv', index=False)
end = time.time()
print('Time taken by the Kernel  {}.'.format(end - fixstart))

"""
LGBM Regression on TfIDF of text features and One-Hot-Encoded Categoricals
Featues based on Alexandu Papiu's (https://www.kaggle.com/apapiu) script: https://www.kaggle.com/apapiu/ridge-script
LGBM based on InfiniteWing's (https://www.kaggle.com/infinitewing) script: https://www.kaggle.com/infinitewing/lightgbm-example
"""

import pandas as pd
import numpy as np
import scipy

from sklearn.linear_model import Ridge, LogisticRegression
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.preprocessing import LabelBinarizer
import lightgbm as lgb
import time

import gc

start = fixstart = time.time()

NUM_BRANDS = 2500
NAME_MIN_DF = 10
MAX_FEAT_DESCP = 50000

print("Reading in Data")

df_train = pd.read_csv('../input/train.tsv', sep='\t')
df_test = pd.read_csv('../input/test.tsv', sep='\t')

df = pd.concat([df_train, df_test], 0)
nrow_train = df_train.shape[0]
y_train = np.log1p(df_train["price"])

del df_train
gc.collect()

print(df.memory_usage(deep = True))

df["category_name"] = df["category_name"].fillna("Other").astype("category")
df["brand_name"] = df["brand_name"].fillna("unknown")

pop_brands = df["brand_name"].value_counts().index[:NUM_BRANDS]
df.loc[~df["brand_name"].isin(pop_brands), "brand_name"] = "Other"

df["item_description"] = df["item_description"].fillna("None")
df["item_condition_id"] = df["item_condition_id"].astype("category")
df["brand_name"] = df["brand_name"].astype("category")

# description contains interesting words
interesting_words = ['new', 'perfect', 'fit', 'used', 'super', 'cute', 'excellent',
                     'great', 'retail', '[rm]', 'never' ]
for word in interesting_words:
    df[word] = df['item_description'].apply(lambda x : word in x.lower())


print(df.memory_usage(deep = True))

print("Encodings")
count = CountVectorizer(min_df=NAME_MIN_DF)
X_name = count.fit_transform(df["name"])

print("Category Encoders")
unique_categories = pd.Series("/".join(df["category_name"].unique().astype("str")).split("/")).unique()
count_category = CountVectorizer()
X_category = count_category.fit_transform(df["category_name"])

print("Descp encoders")
count_descp = TfidfVectorizer(max_features = MAX_FEAT_DESCP, 
                              ngram_range = (1,3),
                              stop_words = "english")
X_descp = count_descp.fit_transform(df["item_description"])

print("Brand encoders")
vect_brand = LabelBinarizer(sparse_output=True)
X_brand = vect_brand.fit_transform(df["brand_name"])

print("Dummy Encoders")
X_dummies = scipy.sparse.csr_matrix(pd.get_dummies(df[[
    "item_condition_id", "shipping"]], sparse = True).values)

X = scipy.sparse.hstack((X_dummies, 
                         X_descp,
                         X_brand,
                         X_category,
                         X_name)).tocsr()

print([X_dummies.shape, X_category.shape, 
       X_name.shape, X_descp.shape, X_brand.shape])

X_train = X[:nrow_train]
X_test = X[nrow_train:]

end = time.time()
print("Time taken reading and encoding  {}.".format((end-start)))

#LGBM Training
start = time.time()

params = {
    'learning_rate': 0.8,
    'application': 'regression',
    'max_depth': 3,
    'num_leaves': 100,
    'verbosity': -1,
    'metric': 'RMSE', 
    'max_bin':8192
}


train_X, valid_X, train_y, valid_y = train_test_split(X_train, y_train, 
                                                      test_size = 0.1, 
                                                      random_state = 144) 
d_train = lgb.Dataset(train_X, label=train_y)
d_valid = lgb.Dataset(valid_X, label=valid_y)
watchlist = [d_train, d_valid]

model = lgb.train(params, train_set=d_train, num_boost_round=3000, 
                  valid_sets=watchlist, 
                  early_stopping_rounds=50, verbose_eval=100) 
preds = model.predict(X_test)
end = time.time()
print("Time taken training LGBM  {}.".format((end-start)))

#Ridge Training
start = time.time()

modelr = Ridge(solver = "lsqr", fit_intercept=False)
modelr.fit(X_train, y_train)

preds += modelr.predict(X_test)
preds /= 2
end = time.time()
print("Time taken training Ridge  {}.".format((end-start)))


df_test["price"] = np.expm1(preds)
df_test[["test_id", "price"]].to_csv("ff_LGBM_Ridge_3.csv", index = False)
end = time.time()
print("Time taken by the Kernel  {}.".format((end-fixstart)))
