import datetime

import pandas as pd
from dateutil import parser

from indicators import *


def train_predictors(market_data, functions_for_typical_price_data, functions_for_hlc_price_data, labels_for_typical_price_data, labels_for_hlc_price_data):
    """
    THIS IS USED TO COMPUTE VALUES OF THE PREDICTORS TO TRAIN A CLASSIFIER/REGRESSOR
    IT ALSO SCALES THE VARIABLE AND RETURNS A DICT GIVING THE SCALING FACTORS
    THESE SCALING FACTORS ARE TO BE USED TO MULTIPLY THE TEST PREDICTORS !!!
    :param market_data: a dataframe obtained thanks to utils.PoloniexMarket.get_historical_data
    :param functions_for_typical_price_data: a list of functions that takes into arguments only a list of typical prices
    :param functions_for_hlc_price_data: a list of functions that takes into arguments only a list of high,low, and close prices
    :param labels_for_typical_price_data: the names of the typical functions
    :param labels_for_hlc_price_data: the names of the HLC functions
    :return: a tuple : (a dataframe that contains the standardized variables used to predict,
                        a dict that contains the scaling factors
                        )
    """
    standard = {}
    # high = market_data.loc[:, 'high'].values.tolist()
    # low = market_data.loc[:, 'low'].values.tolist()
    # close = market_data.loc[:, 'close'].values.tolist()
    volume = market_data.loc[:, 'volume'].values
    # typical_prices = typical_price(high, low, close)
    typical_prices = market_data.loc[:, 'weightedAverage'].values
    standard['volume'] = (np.nanmean(volume), np.nanstd(volume))
    standard['typical_price'] = (np.nanmean(typical_prices), np.nanstd(typical_prices))
    x = ((volume - standard['volume'][0])/standard['volume'][1])
    x = np.c_[(typical_prices - standard['typical_price'][0])/standard['typical_price'][1], x]
    typical_prices = typical_prices.tolist()
    for f, label in zip(functions_for_typical_price_data, labels_for_typical_price_data):
        values = np.array(f(typical_prices))
        standard[label] = (np.nanmean(values), np.nanstd(values))
        x = np.c_[x, (values - standard[label][0])/standard[label][1]]
    # for f, label in zip(functions_for_hlc_price_data, labels_for_hlc_price_data):
    #     values = np.array(f(high, low, close))
    #     if 'typical_price' in label and label != 'typical_price':
    #         standard[label] = standard['typical_price']
    #     else:
    #         standard[label] = (np.nanmean(values), np.nanstd(values))
    #     x = np.c_[x, (values - standard[label][0])/standard[label][1]]
    return pd.DataFrame(data=x, columns=['typical_price', 'volume']+labels_for_typical_price_data, index=market_data.index), standard


def test_predictors(market_data, functions_for_typical_price_data, functions_for_hlc_price_data, labels_for_typical_price_data, labels_for_hlc_price_data, scaling_dict):
    """
    THIS IS USED TO COMPUTE THE PREDICTORS FROM A TEST DATASET
    :param market_data:
    :param functions_for_typical_price_data:
    :param functions_for_hlc_price_data:
    :param labels_for_typical_price_data:
    :param labels_for_hlc_price_data:
    :param scaling_dict: THE VALUES TO BE SUBSTRACTED AND MULTIPLIED TO THE MARKET DATA. THE LATTER MUSTN'T BE SCALED ACCORDING TO ITS OWN STANDARD.
    THIS DICT IS TO BE GENERATED BY THE FUNCTION TRAIN_PREDICTORS (see ABOVE)
    :return:
    """
    x = []
    # high = market_data.loc[:, 'high'].values.tolist()
    # low = market_data.loc[:, 'low'].values.tolist()
    # close = market_data.loc[:, 'close'].values.tolist()
    volume = market_data.loc[:, 'volume'].values
    # typical_prices = typical_price(high, low, close)
    typical_prices = market_data.loc[:, 'weightedAverage'].values
    x.append(((typical_prices - scaling_dict['typical_price'][0])/scaling_dict['typical_price'][1]).tolist())
    typical_prices = typical_prices.tolist()
    x.append(((volume - scaling_dict['volume'][0])/scaling_dict['volume'][1]).tolist())
    for f, label in zip(functions_for_typical_price_data, labels_for_typical_price_data):
        values = np.array(f(typical_prices))
        x.append(((values - scaling_dict[label][0])/scaling_dict[label][1]).tolist())
    # for f, label in zip(functions_for_hlc_price_data, labels_for_hlc_price_data):
    #     values = np.array(f(high, low, close))
    #     x.append(((values - scaling_dict[label][0])/scaling_dict[label][1]).tolist())
    return pd.DataFrame(data=np.array(x).T, columns=['typical_price', 'volume']+labels_for_typical_price_data, index=market_data.index)


def typical_prices_from_trades_history(trade_history, read=False):
    """
    LE PRIX MOYENNE A 10H00 EST LA MOYENNE DES PRIX ENTRE 10H00 ET 10H05 (PONDERES PAR LES VOLUMES)
    :param trade_history: a dataframe you get from PoloniexMarket.get_trade_history
    :param read: if True, it means that you read the data from a csv file. In that case data are unicode
    :return: a tuple containing :
                    - a list of all the dates from the beginning of the period (each 5 minute generally)
                    - a list of the typical prices for these dates
                    - a list of the exchanged volumes for these dates
    """
    if not read:  # in that case you get data directly from the market
        start_date = _get_biggest_anterior_date(trade_history.index[-1]) + datetime.timedelta(minutes=5)
        trade_history = _cut_smallest_dates(trade_history, start_date)
        current_date = start_date + datetime.timedelta(hours=2)
        dates = [start_date + datetime.timedelta(hours=2)]
        # dates = []
        volumes = []
        while current_date < trade_history.index[0] + datetime.timedelta(hours=2):
            current_date += datetime.timedelta(minutes=5)
            dates.append(current_date)
        typical_prices = []
        k = 1
        date_index = 1
        while k < trade_history.shape[0]+1:  # you have to go through all the rows of the dataframe
            price = 0
            normalizing_factor = 0
            volume = 0
            j = 0
            try:
                # you have one price per date. Each price is computed from the traded prices between a date and the following one
                while trade_history.index[-(k+j)] + datetime.timedelta(hours=2) <= dates[date_index]:
                    if isinstance(trade_history.loc[trade_history.index[-(k+j)], 'rate'], pd.Series):  # there can be several trades at the same time. In that case you get a series rather than a number
                        for l in range(len(trade_history.loc[trade_history.index[-(k+j)], 'total'].values.tolist())):
                            price += float(trade_history.loc[trade_history.index[-(k+j)], 'total'].values[l])  # several rows could have the same date. So take the mean. Maybe weight the prices with the volume ?
                            normalizing_factor += float(trade_history.loc[trade_history.index[-(k+j)], 'amount'].values[l])
                            volume += float(trade_history.loc[trade_history.index[-(k+j)], 'total'].values[l])
                        j += trade_history.loc[trade_history.index[-(k+j)], 'rate'].shape[0]
                    else:
                        price += float(trade_history.loc[trade_history.index[-(k+j)], 'total'])
                        normalizing_factor += float(trade_history.loc[trade_history.index[-(k+j)], 'amount'])
                        volume += float(trade_history.loc[trade_history.index[-(k+j)], 'total'])
                        j += 1
                if j != 0:
                    price /= normalizing_factor
                    typical_prices.append(price)
                    k += j
                else:
                    typical_prices.append(np.nan)
                    k += 1
                volumes.append(volume)
                date_index += 1
            except IndexError:
                # print "k+j: %s, \n trade_history.shape: %s, \n date_index: %s, \n len(dates): %s" % (k+j, trade_history.shape, date_index, len(dates))
                break
        return dates, typical_prices, volumes
    else:  # otherwise you get a file where every data is unicode
        start_date = _get_biggest_anterior_date(parser.parse(trade_history.index[-1])) + datetime.timedelta(minutes=5)
        current_date = start_date + datetime.timedelta(hours=2)
        dates = []
        volumes = []
        while current_date < parser.parse(trade_history.index[0]) + datetime.timedelta(hours=2):
            current_date += datetime.timedelta(minutes=5)
            dates.append(current_date)
        typical_prices = []
        k = 1
        date_index = 1
        while k < trade_history.shape[0]+1:
            price = 0
            volume = 0
            normalizing_factor = 0
            j = 0
            try:
                while parser.parse(trade_history.index[-(k+j)]) + datetime.timedelta(hours=2) <= dates[date_index]:
                    if not(isinstance(trade_history.loc[trade_history.index[-(k+j)], 'rate'], np.float64) or isinstance(trade_history.loc[trade_history.index[-(k+j)], 'rate'], np.float32) or isinstance(trade_history.loc[trade_history.index[-(k+j)], 'rate'], float)):
                        price += np.sum(trade_history.loc[trade_history.index[-(k+j)], 'total'].values)  # several rows could have the same date. So take the mean. Maybe weight the prices with the volume ?
                        normalizing_factor += np.sum(trade_history.loc[trade_history.index[-(k+j)], 'amount'].values)
                        volume += np.sum(trade_history.loc[trade_history.index[-(k+j)], 'total'].values)
                        j += trade_history.loc[trade_history.index[-(k+j)], 'rate'].shape[0]
                    else:
                        price += trade_history.loc[trade_history.index[-(k+j)], 'total']
                        normalizing_factor += trade_history.loc[trade_history.index[-(k+j)], 'amount']
                        volume += trade_history.loc[trade_history.index[-(k+j)], 'total']
                        j += 1
                if j != 0:
                    price /= normalizing_factor
                    typical_prices.append(price)
                    k += j
                else:
                    typical_prices.append(np.nan)
                    k += 1
                volumes.append(volume)
                date_index += 1
            except IndexError:
                # print "k+j: %s, \n trade_history.shape: %s, \n date_index: %s, \n len(dates): %s" % (k+j, trade_history.shape, date_index, len(dates))
                break
        return dates, typical_prices, volumes


def predictors_from_typical_prices(typical_prices, volume, scaling_dict, functions=ALL_TYPICAL_FUNCTIONS, names=ALL_TYPICAL_NAMES):
    x = np.array(volume)
    for f, label in zip(functions, names):
        try:
            values = (np.array(f(typical_prices)) - scaling_dict[label][0]) / scaling_dict[label][1]
            x = np.c_[x, np.array(values)]
        except TypeError:
            print "label:%s" % label
            print "prices : %s" %typical_prices
            print "scaling factor : %s" % scaling_dict[label]
            print "value obtained : %s" % f(typical_prices)
            return
    return pd.DataFrame(data=x, index=None, columns=['volume']+ALL_TYPICAL_NAMES)


def datestring_to_timestamp(datestring):
    return datetime_to_timestamp(parser.parse(datestring))


def datestring_to_datetime(datestring):
    return parser.parse(datestring)


def datetime_to_timestamp(dt, epoch=datetime.datetime(1970, 1, 1)):
    td = dt - epoch
    return (td.microseconds + (td.seconds + td.days * 86400) * 10**6) / 10**6


def timestamp_to_datetime(timestamp):
    return parser.parse(datetime.datetime.fromtimestamp(
        timestamp
    ).strftime('%Y-%m-%d %H:%M:%S'))


def parse_predictor_dataframe(predictors):
    begin = 0
    end = -1
    for k in range(predictors.shape[0]):
        found = False
        for element in predictors.loc[predictors.index[k], :].values.tolist():
            if np.isnan(element):
                found=True
                break
        if not found :
            begin = k
            break
    for k in range(1, predictors.shape[0]):
        found = False
        for element in predictors.loc[predictors.index[-k], :].values.tolist():
            if np.isnan(element):
                found=True
                break
        if not found:
            end = -k
            break
    return predictors.loc[predictors.index[begin]:predictors.index[end], :]


def _get_biggest_anterior_date(date) :
    seconds = date.second
    minutes = date.minute%5
    microseconds = date.microsecond
    return date - datetime.timedelta(seconds=seconds, minutes=minutes, microseconds=microseconds)


def _cut_smallest_dates(trade_history, smallest_date_limit):
    k = -1
    while trade_history.index[k] < smallest_date_limit:
        k -= 1
    return trade_history.loc[:trade_history.index[k], :]


bdf3c19d376416252eb1fe7ac50d0b576da6ff45
import datetime

import pandas as pd
from dateutil import parser

from indicators import *


def train_predictors(market_data, functions_for_typical_price_data, functions_for_hlc_price_data, labels_for_typical_price_data, labels_for_hlc_price_data):
    """
    THIS IS USED TO COMPUTE VALUES OF THE PREDICTORS TO TRAIN A CLASSIFIER/REGRESSOR
    IT ALSO SCALES THE VARIABLE AND RETURNS A DICT GIVING THE SCALING FACTORS
    THESE SCALING FACTORS ARE TO BE USED TO MULTIPLY THE TEST PREDICTORS !!!
    :param market_data: a dataframe obtained thanks to utils.PoloniexMarket.get_historical_data
    :param functions_for_typical_price_data: a list of functions that takes into arguments only a list of typical prices
    :param functions_for_hlc_price_data: a list of functions that takes into arguments only a list of high,low, and close prices
    :param labels_for_typical_price_data: the names of the typical functions
    :param labels_for_hlc_price_data: the names of the HLC functions
    :return: a tuple : (a dataframe that contains the standardized variables used to predict,
                        a dict that contains the scaling factors
                        )
    """
    standard = {}
    # high = market_data.loc[:, 'high'].values.tolist()
    # low = market_data.loc[:, 'low'].values.tolist()
    # close = market_data.loc[:, 'close'].values.tolist()
    volume = market_data.loc[:, 'volume'].values
    # typical_prices = typical_price(high, low, close)
    typical_prices = market_data.loc[:, 'weightedAverage'].values
    standard['volume'] = (np.nanmean(volume), np.nanstd(volume))
    standard['typical_price'] = (np.nanmean(typical_prices), np.nanstd(typical_prices))
    x = ((volume - standard['volume'][0])/standard['volume'][1])
    x = np.c_[(typical_prices - standard['typical_price'][0])/standard['typical_price'][1], x]
    typical_prices = typical_prices.tolist()
    for f, label in zip(functions_for_typical_price_data, labels_for_typical_price_data):
        values = np.array(f(typical_prices))
        standard[label] = (np.nanmean(values), np.nanstd(values))
        x = np.c_[x, (values - standard[label][0])/standard[label][1]]
    # for f, label in zip(functions_for_hlc_price_data, labels_for_hlc_price_data):
    #     values = np.array(f(high, low, close))
    #     if 'typical_price' in label and label != 'typical_price':
    #         standard[label] = standard['typical_price']
    #     else:
    #         standard[label] = (np.nanmean(values), np.nanstd(values))
    #     x = np.c_[x, (values - standard[label][0])/standard[label][1]]
    return pd.DataFrame(data=x, columns=['typical_price', 'volume']+labels_for_typical_price_data, index=market_data.index), standard


def test_predictors(market_data, functions_for_typical_price_data, functions_for_hlc_price_data, labels_for_typical_price_data, labels_for_hlc_price_data, scaling_dict):
    """
    THIS IS USED TO COMPUTE THE PREDICTORS FROM A TEST DATASET
    :param market_data:
    :param functions_for_typical_price_data:
    :param functions_for_hlc_price_data:
    :param labels_for_typical_price_data:
    :param labels_for_hlc_price_data:
    :param scaling_dict: THE VALUES TO BE SUBSTRACTED AND MULTIPLIED TO THE MARKET DATA. THE LATTER MUSTN'T BE SCALED ACCORDING TO ITS OWN STANDARD.
    THIS DICT IS TO BE GENERATED BY THE FUNCTION TRAIN_PREDICTORS (see ABOVE)
    :return:
    """
    x = []
    # high = market_data.loc[:, 'high'].values.tolist()
    # low = market_data.loc[:, 'low'].values.tolist()
    # close = market_data.loc[:, 'close'].values.tolist()
    volume = market_data.loc[:, 'volume'].values
    # typical_prices = typical_price(high, low, close)
    typical_prices = market_data.loc[:, 'weightedAverage'].values
    x.append(((typical_prices - scaling_dict['typical_price'][0])/scaling_dict['typical_price'][1]).tolist())
    typical_prices = typical_prices.tolist()
    x.append(((volume - scaling_dict['volume'][0])/scaling_dict['volume'][1]).tolist())
    for f, label in zip(functions_for_typical_price_data, labels_for_typical_price_data):
        values = np.array(f(typical_prices))
        x.append(((values - scaling_dict[label][0])/scaling_dict[label][1]).tolist())
    # for f, label in zip(functions_for_hlc_price_data, labels_for_hlc_price_data):
    #     values = np.array(f(high, low, close))
    #     x.append(((values - scaling_dict[label][0])/scaling_dict[label][1]).tolist())
    return pd.DataFrame(data=np.array(x).T, columns=['typical_price', 'volume']+labels_for_typical_price_data, index=market_data.index)


def typical_prices_from_trades_history(trade_history, read=False):
    """
    LE PRIX MOYENNE A 10H00 EST LA MOYENNE DES PRIX ENTRE 10H00 ET 10H05 (PONDERES PAR LES VOLUMES)
    :param trade_history: a dataframe you get from PoloniexMarket.get_trade_history
    :param read: if True, it means that you read the data from a csv file. In that case data are unicode
    :return: a tuple containing :
                    - a list of all the dates from the beginning of the period (each 5 minute generally)
                    - a list of the typical prices for these dates
                    - a list of the exchanged volumes for these dates
    """
    if not read:  # in that case you get data directly from the market
        start_date = _get_biggest_anterior_date(trade_history.index[-1]) + datetime.timedelta(minutes=5)
        trade_history = _cut_smallest_dates(trade_history, start_date)
        current_date = start_date + datetime.timedelta(hours=2)
        dates = [start_date + datetime.timedelta(hours=2)]
        # dates = []
        volumes = []
        while current_date < trade_history.index[0] + datetime.timedelta(hours=2):
            current_date += datetime.timedelta(minutes=5)
            dates.append(current_date)
        typical_prices = []
        k = 1
        date_index = 1
        while k < trade_history.shape[0]+1:  # you have to go through all the rows of the dataframe
            price = 0
            normalizing_factor = 0
            volume = 0
            j = 0
            try:
                # you have one price per date. Each price is computed from the traded prices between a date and the following one
                while trade_history.index[-(k+j)] + datetime.timedelta(hours=2) <= dates[date_index]:
                    if isinstance(trade_history.loc[trade_history.index[-(k+j)], 'rate'], pd.Series):  # there can be several trades at the same time. In that case you get a series rather than a number
                        for l in range(len(trade_history.loc[trade_history.index[-(k+j)], 'total'].values.tolist())):
                            price += float(trade_history.loc[trade_history.index[-(k+j)], 'total'].values[l])  # several rows could have the same date. So take the mean. Maybe weight the prices with the volume ?
                            normalizing_factor += float(trade_history.loc[trade_history.index[-(k+j)], 'amount'].values[l])
                            volume += float(trade_history.loc[trade_history.index[-(k+j)], 'total'].values[l])
                        j += trade_history.loc[trade_history.index[-(k+j)], 'rate'].shape[0]
                    else:
                        price += float(trade_history.loc[trade_history.index[-(k+j)], 'total'])
                        normalizing_factor += float(trade_history.loc[trade_history.index[-(k+j)], 'amount'])
                        volume += float(trade_history.loc[trade_history.index[-(k+j)], 'total'])
                        j += 1
                if j != 0:
                    price /= normalizing_factor
                    typical_prices.append(price)
                    k += j
                else:
                    typical_prices.append(np.nan)
                    k += 1
                volumes.append(volume)
                date_index += 1
            except IndexError:
                # print "k+j: %s, \n trade_history.shape: %s, \n date_index: %s, \n len(dates): %s" % (k+j, trade_history.shape, date_index, len(dates))
                break
        return dates, typical_prices, volumes
    else:  # otherwise you get a file where every data is unicode
        start_date = _get_biggest_anterior_date(parser.parse(trade_history.index[-1])) + datetime.timedelta(minutes=5)
        current_date = start_date + datetime.timedelta(hours=2)
        dates = []
        volumes = []
        while current_date < parser.parse(trade_history.index[0]) + datetime.timedelta(hours=2):
            current_date += datetime.timedelta(minutes=5)
            dates.append(current_date)
        typical_prices = []
        k = 1
        date_index = 1
        while k < trade_history.shape[0]+1:
            price = 0
            volume = 0
            normalizing_factor = 0
            j = 0
            try:
                while parser.parse(trade_history.index[-(k+j)]) + datetime.timedelta(hours=2) <= dates[date_index]:
                    if not(isinstance(trade_history.loc[trade_history.index[-(k+j)], 'rate'], np.float64) or isinstance(trade_history.loc[trade_history.index[-(k+j)], 'rate'], np.float32) or isinstance(trade_history.loc[trade_history.index[-(k+j)], 'rate'], float)):
                        price += np.sum(trade_history.loc[trade_history.index[-(k+j)], 'total'].values)  # several rows could have the same date. So take the mean. Maybe weight the prices with the volume ?
                        normalizing_factor += np.sum(trade_history.loc[trade_history.index[-(k+j)], 'amount'].values)
                        volume += np.sum(trade_history.loc[trade_history.index[-(k+j)], 'total'].values)
                        j += trade_history.loc[trade_history.index[-(k+j)], 'rate'].shape[0]
                    else:
                        price += trade_history.loc[trade_history.index[-(k+j)], 'total']
                        normalizing_factor += trade_history.loc[trade_history.index[-(k+j)], 'amount']
                        volume += trade_history.loc[trade_history.index[-(k+j)], 'total']
                        j += 1
                if j != 0:
                    price /= normalizing_factor
                    typical_prices.append(price)
                    k += j
                else:
                    typical_prices.append(np.nan)
                    k += 1
                volumes.append(volume)
                date_index += 1
            except IndexError:
                # print "k+j: %s, \n trade_history.shape: %s, \n date_index: %s, \n len(dates): %s" % (k+j, trade_history.shape, date_index, len(dates))
                break
        return dates, typical_prices, volumes


def predictors_from_typical_prices(typical_prices, volume, scaling_dict, functions=ALL_TYPICAL_FUNCTIONS, names=ALL_TYPICAL_NAMES):
    x = np.array(volume)
    for f, label in zip(functions, names):
        try:
            values = (np.array(f(typical_prices)) - scaling_dict[label][0]) / scaling_dict[label][1]
            x = np.c_[x, np.array(values)]
        except TypeError:
            print "label:%s" % label
            print "prices : %s" %typical_prices
            print "scaling factor : %s" % scaling_dict[label]
            print "value obtained : %s" % f(typical_prices)
            return
    return pd.DataFrame(data=x, index=None, columns=['volume']+ALL_TYPICAL_NAMES)


def datestring_to_timestamp(datestring):
    return datetime_to_timestamp(parser.parse(datestring))


def datestring_to_datetime(datestring):
    return parser.parse(datestring)


def datetime_to_timestamp(dt, epoch=datetime.datetime(1970, 1, 1)):
    td = dt - epoch
    return (td.microseconds + (td.seconds + td.days * 86400) * 10**6) / 10**6


def timestamp_to_datetime(timestamp):
    return parser.parse(datetime.datetime.fromtimestamp(
        timestamp
    ).strftime('%Y-%m-%d %H:%M:%S'))


def parse_predictor_dataframe(predictors):
    begin = 0
    end = -1
    for k in range(predictors.shape[0]):
        found = False
        for element in predictors.loc[predictors.index[k], :].values.tolist():
            if np.isnan(element):
                found=True
                break
        if not found :
            begin = k
            break
    for k in range(1, predictors.shape[0]):
        found = False
        for element in predictors.loc[predictors.index[-k], :].values.tolist():
            if np.isnan(element):
                found=True
                break
        if not found:
            end = -k
            break
    return predictors.loc[predictors.index[begin]:predictors.index[end], :]


def _get_biggest_anterior_date(date) :
    seconds = date.second
    minutes = date.minute%5
    microseconds = date.microsecond
    return date - datetime.timedelta(seconds=seconds, minutes=minutes, microseconds=microseconds)


def _cut_smallest_dates(trade_history, smallest_date_limit):
    k = -1
    while trade_history.index[k] < smallest_date_limit:
        k -= 1
    return trade_history.loc[:trade_history.index[k], :]

