import sys
import os
import pandas as pd
import numpy as np
import pickle
from sqlalchemy import create_engine
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.metrics import classification_report
from sklearn.model_selection import GridSearchCV

def load_data(database_filepath):
    """
    Loading data form database

    Parameters:
    database_filepath(str): the file path of database file

    Output:
    X(array): message
    Y(ndarray): categories
    category_names(list): category names
    """
    path=os.path.abspath( database_filepath ) 
    engine = create_engine('sqlite:///'+path)
    df = pd.read_sql_table("DisasterResponse",engine)
    X = df.message.values
    Y = df.iloc[:,4:].values
    category_names=df.iloc[:,4:].columns
    
    return X,Y,category_names


def tokenize(text):
    """
    Tokenize text into words

    Parameters:
    text(str): a text to be tokenized

    Output:
    clear_tokens(list): a list of words 
    """
    tokens = word_tokenize(text)
    lemmatizer = WordNetLemmatizer()

    clean_tokens = []
    for tok in tokens:
        clean_tok = lemmatizer.lemmatize(tok).lower().strip()
        clean_tokens.append(clean_tok)

    return clean_tokens


def build_model():

	"""
	Building a gridsearch model

	Parameters:
	None

	Outputs:
	cv(gridseachcv object): a model of gridsearch

	"""

    pipeline = Pipeline([
        ('vect', CountVectorizer(tokenizer=tokenize)),
        ('tfidf', TfidfTransformer()),
        ('clf', RandomForestClassifier())
    ])

    parameters = {
        'vect__ngram_range': ((1, 2),),
        'vect__max_df': (0.5, ),
        'vect__max_features': ( 5000, ),
        'tfidf__use_idf': (False,),
        'clf__n_estimators': [100, ],
        'clf__min_samples_split': [2,],
    }


    cv = GridSearchCV(pipeline, param_grid=parameters)

    return cv


def evaluate_model(model, X_test, Y_test, category_names):
	'''
	evaluate the model, and print the accuracy report of each category

	Parameters:
	model: the gridseach model
	X_test(2darray): X testing data
	Y_test(2darray): Y testing data
	category_names(list): name list of all categories
	'''

    y_pred = model.predict(X_test)

    for i,name in enumerate(category_names):
        print(name)
        print(classification_report(Y_test[:,i], y_pred[:,i]))


def save_model(model, model_filepath):
	"""
	save the trained model into pickle file

	Parameter:
	model: the trained model
	model_filepath(str): the path of the pickle file which is going to be saved
	"""
    with open(model_filepath, 'wb') as f:
        pickle.dump(model, f)



def main():
    if len(sys.argv) == 3:
        database_filepath, model_filepath = sys.argv[1:]
        print('Loading data...\n    DATABASE: {}'.format(database_filepath))
        X, Y, category_names = load_data(database_filepath)
        X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)
        
        print('Building model...')
        model = build_model()
        
        print('Training model...')
        model.fit(X_train, Y_train)
        
        print('Evaluating model...')
        evaluate_model(model, X_test, Y_test, category_names)

        print('Saving model...\n    MODEL: {}'.format(model_filepath))
        save_model(model, model_filepath)

        print('Trained model saved!')

    else:
        print('Please provide the filepath of the disaster messages database '\
              'as the first argument and the filepath of the pickle file to '\
              'save the model to as the second argument. \n\nExample: python '\
              'train_classifier.py ../data/DisasterResponse.db classifier.pkl')


if __name__ == '__main__':
    main()
8b47a5fbf4e5d06b22efccf1aad465d7c2a3228a
import sys
import os
import pandas as pd
import numpy as np
import pickle
from sqlalchemy import create_engine
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.metrics import classification_report
from sklearn.model_selection import GridSearchCV

def load_data(database_filepath):
    """
    Loading data form database

    Parameters:
    database_filepath(str): the file path of database file

    Output:
    X(array): message
    Y(ndarray): categories
    category_names(list): category names
    """
    path=os.path.abspath( database_filepath ) 
    engine = create_engine('sqlite:///'+path)
    df = pd.read_sql_table("DisasterResponse",engine)
    X = df.message.values
    Y = df.iloc[:,4:].values
    category_names=df.iloc[:,4:].columns
    
    return X,Y,category_names


def tokenize(text):
    """
    Tokenize text into words

    Parameters:
    text(str): a text to be tokenized

    Output:
    clear_tokens(list): a list of words 
    """
    tokens = word_tokenize(text)
    lemmatizer = WordNetLemmatizer()

    clean_tokens = []
    for tok in tokens:
        clean_tok = lemmatizer.lemmatize(tok).lower().strip()
        clean_tokens.append(clean_tok)

    return clean_tokens


def build_model():

	"""
	Building a gridsearch model

	Parameters:
	None

	Outputs:
	cv(gridseachcv object): a model of gridsearch

	"""

    pipeline = Pipeline([
        ('vect', CountVectorizer(tokenizer=tokenize)),
        ('tfidf', TfidfTransformer()),
        ('clf', RandomForestClassifier())
    ])

    parameters = {
        'vect__ngram_range': ((1, 2),),
        'vect__max_df': (0.5, ),
        'vect__max_features': ( 5000, ),
        'tfidf__use_idf': (False,),
        'clf__n_estimators': [100, ],
        'clf__min_samples_split': [2,],
    }


    cv = GridSearchCV(pipeline, param_grid=parameters)

    return cv


def evaluate_model(model, X_test, Y_test, category_names):
	'''
	evaluate the model, and print the accuracy report of each category

	Parameters:
	model: the gridseach model
	X_test(2darray): X testing data
	Y_test(2darray): Y testing data
	category_names(list): name list of all categories
	'''

    y_pred = model.predict(X_test)

    for i,name in enumerate(category_names):
        print(name)
        print(classification_report(Y_test[:,i], y_pred[:,i]))


def save_model(model, model_filepath):
	"""
	save the trained model into pickle file

	Parameter:
	model: the trained model
	model_filepath(str): the path of the pickle file which is going to be saved
	"""
    with open(model_filepath, 'wb') as f:
        pickle.dump(model, f)



def main():
    if len(sys.argv) == 3:
        database_filepath, model_filepath = sys.argv[1:]
        print('Loading data...\n    DATABASE: {}'.format(database_filepath))
        X, Y, category_names = load_data(database_filepath)
        X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)
        
        print('Building model...')
        model = build_model()
        
        print('Training model...')
        model.fit(X_train, Y_train)
        
        print('Evaluating model...')
        evaluate_model(model, X_test, Y_test, category_names)

        print('Saving model...\n    MODEL: {}'.format(model_filepath))
        save_model(model, model_filepath)

        print('Trained model saved!')

    else:
        print('Please provide the filepath of the disaster messages database '\
              'as the first argument and the filepath of the pickle file to '\
              'save the model to as the second argument. \n\nExample: python '\
              'train_classifier.py ../data/DisasterResponse.db classifier.pkl')


if __name__ == '__main__':
    main()