# -*-coding: utf-8 -*-
# ref scikit-learn.org/stable/tutorial/machine_learning_map/index.html
# ref https://pythonprogramming.net/getting-data-sp-500-index-value-comparison
#           /?completed=/using-pandas-structure-process-data/
# 1. data
# 2. labeling
# 3. choose right machine learning algorithm

# stock fundemental
# Parsing data
import pandas as pd
import os   # interact with directories
import time
from datetime import datetime
# from time import mktime

# import matplotlib
# import matplotlib.pyplot as plt

import re
# import urllib

from matplotlib import style
style.use('dark_background')


# homepath = "/Users/jihwanpark/PythonML"
homepath = "/Volumes/SD64/Dropbox/200.CaptainResearch/Python_MachineLearning/PythonML"
# homepath = "/Users/ehmp/Dropbox/200.CaptainResearch/Python_MachineLearning/PythonML"

path = homepath+"/intraQuarter"


def Key_Stats(gather=["Total Debt/Equity",
                      'Trailing P/E',
                      'Price/Sales',
                      'Price/Book',
                      'Profit Margin',
                      'Operating Margin',
                      'Return on Assets',
                      'Return on Equity',
                      'Revenue Per Share',
                      'Market Cap',
                      'Enterprise Value',
                      'Forward P/E',
                      'PEG Ratio',
                      'Enterprise Value/Revenue',
                      'Enterprise Value/EBITDA',
                      'Revenue',
                      'Gross Profit',
                      'EBITDA',
                      'Net Income Avl to Common ',
                      'Diluted EPS',
                      'Earnings Growth',
                      'Revenue Growth',
                      'Total Cash',
                      'Total Cash Per Share',
                      'Total Debt',
                      'Current Ratio',
                      'Book Value Per Share',
                      'Cash Flow',
                      'Beta',
                      'Held by Insiders',
                      'Held by Institutions',
                      'Shares Short (as of',
                      'Short Ratio',
                      'Short % of Float',
                      'Shares Short (prior ']):

    statspath = path+'/_KeyStats'   # stats directory
    stock_list = [x[0] for x in os.walk(statspath)]

    dfSequence = ['Date',
                               'Unix',
                               'Ticker',
                               'DE Ratio',
                               'Price',
                               'stock_p_change',
                               'SP500',
                               'sp500_p_change',
                               'Difference',
                               'DE Ratio',
                               'Trailing P/E',
                               'Price/Sales',
                               'Price/Book',
                               'Profit Margin',
                               'Operating Margin',
                               'Return on Assets',
                               'Return on Equity',
                               'Revenue Per Share',
                               'Market Cap',
                               'Enterprise Value',
                               'Forward P/E',
                               'PEG Ratio',
                               'Enterprise Value/Revenue',
                               'Enterprise Value/EBITDA',
                               'Revenue',
                               'Gross Profit',
                               'EBITDA',
                               'Net Income Avl to Common ',
                               'Diluted EPS',
                               'Earnings Growth',
                               'Revenue Growth',
                               'Total Cash',
                               'Total Cash Per Share',
                               'Total Debt',
                               'Current Ratio',
                               'Book Value Per Share',
                               'Cash Flow',
                               'Beta',
                               'Held by Insiders',
                               'Held by Institutions',
                               'Shares Short (as of',
                               'Short Ratio',
                               'Short % of Float',
                               'Shares Short (prior ',
                               'Status']

    df = pd.DataFrame(columns=['Date',
                               'Unix',
                               'Ticker',
                               'DE Ratio',
                               'Price',
                               'stock_p_change',
                               'SP500',
                               'sp500_p_change',
                               'Difference',
                               'DE Ratio',
                               'Trailing P/E',
                               'Price/Sales',
                               'Price/Book',
                               'Profit Margin',
                               'Operating Margin',
                               'Return on Assets',
                               'Return on Equity',
                               'Revenue Per Share',
                               'Market Cap',
                               'Enterprise Value',
                               'Forward P/E',
                               'PEG Ratio',
                               'Enterprise Value/Revenue',
                               'Enterprise Value/EBITDA',
                               'Revenue',
                               'Gross Profit',
                               'EBITDA',
                               'Net Income Avl to Common ',
                               'Diluted EPS',
                               'Earnings Growth',
                               'Revenue Growth',
                               'Total Cash',
                               'Total Cash Per Share',
                               'Total Debt',
                               'Current Ratio',
                               'Book Value Per Share',
                               'Cash Flow',
                               'Beta',
                               'Held by Insiders',
                               'Held by Institutions',
                               'Shares Short (as of',
                               'Short Ratio',
                               'Short % of Float',
                               'Shares Short (prior ',
                               'Status'])

    sp500_df = pd.DataFrame.from_csv(homepath+"/YAHOO-INDEX_GSPC.csv")

    ticker_list = []

    for each_dir in stock_list[1:]:  # 25 is test element number
        each_file = os.listdir(each_dir)

        dir_split = each_dir.split("/")
        ticker = dir_split[len(dir_split)-1]
        ticker_list.append(ticker)
        starting_stock_value = False
        starting_sp500_value = False

        if len(each_file) > 0:

            for file in each_file:

                date_stamp = datetime.strptime(file, '%Y%m%d%H%M%S.html')
                unix_time = time.mktime(date_stamp.timetuple())
                full_file_path = each_dir+'/'+file
                source = open(full_file_path, 'r').read()
                try:
                    value_list = []

                    for each_data in gather:
                        try:
                            regex = re.escape(each_data) + r'.*?(\d{1,8}\.\d{1,8}M?B?|N/A)%?</td>'
                            value = re.search(regex, source)
                            value = (value.group(1))

                            if "B" in value:
                                value = float(value.replace("B", ''))*1000000000
                            elif "M" in value:
                                value = float(value.replace("M", ''))*1000000

                            value_list.append(value)

                        except Exception as e:
                            value = "N/A"
                            value_list.append(value)
                    # print(value_list)
                    try:
                        sp500_date = datetime.fromtimestamp(unix_time)
                        sp500_date = sp500_date.strftime('%Y-%m-%d')
                        row = sp500_df[(sp500_df.index == sp500_date)]
                        sp500_value = float(row["Adjusted Close"])
                    except:
                        sp500_time = unix_time
                        lenRow = 0
                        while lenRow == 0:
                            sp500_time = sp500_time - 86400
                            sp500_date = datetime.fromtimestamp(sp500_time)
                            sp500_date = sp500_date.strftime('%Y-%m-%d')
                            row = sp500_df[(sp500_df.index == sp500_date)]
                            lenRow = len(row)
                        sp500_value = float(row["Adjusted Close"])

                    # get stock price
                    try:
                        stock_price = float(source.split('</small><big><b>')[1].split('</b></big>')[0])
                    except Exception as e:
                        #    <span id="yfs_l10_afl">43.27</span>
                        try:
                            stock_price = (source.split('</small><big><b>')[1].split('</b></big>')[0])
                            stock_price = re.search(r'(\d{1,8}\.\d{1,8})', stock_price)
                            stock_price = float(stock_price.group(1))

                            # print(stock_price)
                        except Exception as e:
                            try:
                                stock_price = (source.split('<span class="time_rtq_ticker">')[1].split('</span>')[0])
                                stock_price = re.search(r'(\d{1,8}\.\d{1,8})', stock_price)
                                stock_price = float(stock_price.group(1))
                            except Exception as e:
                                print(str(e), 'a;lsdkfh', file, ticker)

                            # print('Latest:',stock_price)

                            # print('stock price',str(e),ticker,file)
                            # time.sleep(15)

                    # print("stock_price:",stock_price,"ticker:", ticker)
                    if not starting_stock_value:
                        starting_stock_value = stock_price
                    if not starting_sp500_value:
                        starting_sp500_value = sp500_value

                    stock_p_change = ((stock_price - starting_stock_value)
                                      / starting_stock_value) * 100
                    sp500_p_change = ((sp500_value - starting_sp500_value)
                                      / starting_sp500_value) * 100

                    difference = stock_p_change - sp500_p_change
                    if difference > 0:
                        status = "outperform"
                    else:
                        status = "underperform"

                    if value_list.count("N/A") > 0:
                        pass
                    else:
                        # df = df.append({
                        df2 = pd.DataFrame({
                           'Date': date_stamp,
                           'Unix': unix_time,
                           'Ticker': ticker,
                           'Price': stock_price,
                           'stock_p_change': stock_p_change,
                           'SP500': sp500_value,
                           'sp500_p_change': sp500_p_change,
                           'Difference': difference,
                           'DE Ratio': value_list[0],
                           'Trailing P/E': value_list[1],
                           'Price/Sales': value_list[2],
                           'Price/Book': value_list[3],
                           'Profit Margin': value_list[4],
                           'Operating Margin': value_list[5],
                           'Return on Assets': value_list[6],
                           'Return on Equity': value_list[7],
                           'Revenue Per Share': value_list[8],
                           'Market Cap': value_list[9],
                           'Enterprise Value': value_list[10],
                           'Forward P/E': value_list[11],
                           'PEG Ratio': value_list[12],
                           'Enterprise Value/Revenue': value_list[13],
                           'Enterprise Value/EBITDA': value_list[14],
                           'Revenue': value_list[15],
                           'Gross Profit': value_list[16],
                           'EBITDA': value_list[17],
                           'Net Income Avl to Common ': value_list[18],
                           'Diluted EPS': value_list[19],
                           'Earnings Growth': value_list[20],
                           'Revenue Growth': value_list[21],
                           'Total Cash': value_list[22],
                           'Total Cash Per Share': value_list[23],
                           'Total Debt': value_list[24],
                           'Current Ratio': value_list[25],
                           'Book Value Per Share': value_list[26],
                           'Cash Flow': value_list[27],
                           'Beta': value_list[28],
                           'Held by Insiders': value_list[29],
                           'Held by Institutions': value_list[30],
                           'Shares Short (as of': value_list[31],
                           'Short Ratio': value_list[32],
                           'Short % of Float': value_list[33],
                           'Shares Short (prior ': value_list[34],
                           'Status': status}, index=[0])
                        df2 = df2.reindex(columns=dfSequence)
                        df = df.append(df2, ignore_index=True)

                except Exception as e:
                    print(e)
                    # pass

    df.to_csv("key_stats.csv")


Key_Stats()

813430947171a008ed21ee83390441e3be7052cb
<mask token>


def Key_Stats(gather=['Total Debt/Equity', 'Trailing P/E', 'Price/Sales',
    'Price/Book', 'Profit Margin', 'Operating Margin', 'Return on Assets',
    'Return on Equity', 'Revenue Per Share', 'Market Cap',
    'Enterprise Value', 'Forward P/E', 'PEG Ratio',
    'Enterprise Value/Revenue', 'Enterprise Value/EBITDA', 'Revenue',
    'Gross Profit', 'EBITDA', 'Net Income Avl to Common ', 'Diluted EPS',
    'Earnings Growth', 'Revenue Growth', 'Total Cash',
    'Total Cash Per Share', 'Total Debt', 'Current Ratio',
    'Book Value Per Share', 'Cash Flow', 'Beta', 'Held by Insiders',
    'Held by Institutions', 'Shares Short (as of', 'Short Ratio',
    'Short % of Float', 'Shares Short (prior ']):
    statspath = path + '/_KeyStats'
    stock_list = [x[0] for x in os.walk(statspath)]
    dfSequence = ['Date', 'Unix', 'Ticker', 'DE Ratio', 'Price',
        'stock_p_change', 'SP500', 'sp500_p_change', 'Difference',
        'DE Ratio', 'Trailing P/E', 'Price/Sales', 'Price/Book',
        'Profit Margin', 'Operating Margin', 'Return on Assets',
        'Return on Equity', 'Revenue Per Share', 'Market Cap',
        'Enterprise Value', 'Forward P/E', 'PEG Ratio',
        'Enterprise Value/Revenue', 'Enterprise Value/EBITDA', 'Revenue',
        'Gross Profit', 'EBITDA', 'Net Income Avl to Common ',
        'Diluted EPS', 'Earnings Growth', 'Revenue Growth', 'Total Cash',
        'Total Cash Per Share', 'Total Debt', 'Current Ratio',
        'Book Value Per Share', 'Cash Flow', 'Beta', 'Held by Insiders',
        'Held by Institutions', 'Shares Short (as of', 'Short Ratio',
        'Short % of Float', 'Shares Short (prior ', 'Status']
    df = pd.DataFrame(columns=['Date', 'Unix', 'Ticker', 'DE Ratio',
        'Price', 'stock_p_change', 'SP500', 'sp500_p_change', 'Difference',
        'DE Ratio', 'Trailing P/E', 'Price/Sales', 'Price/Book',
        'Profit Margin', 'Operating Margin', 'Return on Assets',
        'Return on Equity', 'Revenue Per Share', 'Market Cap',
        'Enterprise Value', 'Forward P/E', 'PEG Ratio',
        'Enterprise Value/Revenue', 'Enterprise Value/EBITDA', 'Revenue',
        'Gross Profit', 'EBITDA', 'Net Income Avl to Common ',
        'Diluted EPS', 'Earnings Growth', 'Revenue Growth', 'Total Cash',
        'Total Cash Per Share', 'Total Debt', 'Current Ratio',
        'Book Value Per Share', 'Cash Flow', 'Beta', 'Held by Insiders',
        'Held by Institutions', 'Shares Short (as of', 'Short Ratio',
        'Short % of Float', 'Shares Short (prior ', 'Status'])
    sp500_df = pd.DataFrame.from_csv(homepath + '/YAHOO-INDEX_GSPC.csv')
    ticker_list = []
    for each_dir in stock_list[1:]:
        each_file = os.listdir(each_dir)
        dir_split = each_dir.split('/')
        ticker = dir_split[len(dir_split) - 1]
        ticker_list.append(ticker)
        starting_stock_value = False
        starting_sp500_value = False
        if len(each_file) > 0:
            for file in each_file:
                date_stamp = datetime.strptime(file, '%Y%m%d%H%M%S.html')
                unix_time = time.mktime(date_stamp.timetuple())
                full_file_path = each_dir + '/' + file
                source = open(full_file_path, 'r').read()
                try:
                    value_list = []
                    for each_data in gather:
                        try:
                            regex = re.escape(each_data
                                ) + '.*?(\\d{1,8}\\.\\d{1,8}M?B?|N/A)%?</td>'
                            value = re.search(regex, source)
                            value = value.group(1)
                            if 'B' in value:
                                value = float(value.replace('B', '')
                                    ) * 1000000000
                            elif 'M' in value:
                                value = float(value.replace('M', '')) * 1000000
                            value_list.append(value)
                        except Exception as e:
                            value = 'N/A'
                            value_list.append(value)
                    try:
                        sp500_date = datetime.fromtimestamp(unix_time)
                        sp500_date = sp500_date.strftime('%Y-%m-%d')
                        row = sp500_df[sp500_df.index == sp500_date]
                        sp500_value = float(row['Adjusted Close'])
                    except:
                        sp500_time = unix_time
                        lenRow = 0
                        while lenRow == 0:
                            sp500_time = sp500_time - 86400
                            sp500_date = datetime.fromtimestamp(sp500_time)
                            sp500_date = sp500_date.strftime('%Y-%m-%d')
                            row = sp500_df[sp500_df.index == sp500_date]
                            lenRow = len(row)
                        sp500_value = float(row['Adjusted Close'])
                    try:
                        stock_price = float(source.split('</small><big><b>'
                            )[1].split('</b></big>')[0])
                    except Exception as e:
                        try:
                            stock_price = source.split('</small><big><b>')[1
                                ].split('</b></big>')[0]
                            stock_price = re.search('(\\d{1,8}\\.\\d{1,8})',
                                stock_price)
                            stock_price = float(stock_price.group(1))
                        except Exception as e:
                            try:
                                stock_price = source.split(
                                    '<span class="time_rtq_ticker">')[1].split(
                                    '</span>')[0]
                                stock_price = re.search('(\\d{1,8}\\.\\d{1,8})'
                                    , stock_price)
                                stock_price = float(stock_price.group(1))
                            except Exception as e:
                                print(str(e), 'a;lsdkfh', file, ticker)
                    if not starting_stock_value:
                        starting_stock_value = stock_price
                    if not starting_sp500_value:
                        starting_sp500_value = sp500_value
                    stock_p_change = (stock_price - starting_stock_value
                        ) / starting_stock_value * 100
                    sp500_p_change = (sp500_value - starting_sp500_value
                        ) / starting_sp500_value * 100
                    difference = stock_p_change - sp500_p_change
                    if difference > 0:
                        status = 'outperform'
                    else:
                        status = 'underperform'
                    if value_list.count('N/A') > 0:
                        pass
                    else:
                        df2 = pd.DataFrame({'Date': date_stamp, 'Unix':
                            unix_time, 'Ticker': ticker, 'Price':
                            stock_price, 'stock_p_change': stock_p_change,
                            'SP500': sp500_value, 'sp500_p_change':
                            sp500_p_change, 'Difference': difference,
                            'DE Ratio': value_list[0], 'Trailing P/E':
                            value_list[1], 'Price/Sales': value_list[2],
                            'Price/Book': value_list[3], 'Profit Margin':
                            value_list[4], 'Operating Margin': value_list[5
                            ], 'Return on Assets': value_list[6],
                            'Return on Equity': value_list[7],
                            'Revenue Per Share': value_list[8],
                            'Market Cap': value_list[9], 'Enterprise Value':
                            value_list[10], 'Forward P/E': value_list[11],
                            'PEG Ratio': value_list[12],
                            'Enterprise Value/Revenue': value_list[13],
                            'Enterprise Value/EBITDA': value_list[14],
                            'Revenue': value_list[15], 'Gross Profit':
                            value_list[16], 'EBITDA': value_list[17],
                            'Net Income Avl to Common ': value_list[18],
                            'Diluted EPS': value_list[19],
                            'Earnings Growth': value_list[20],
                            'Revenue Growth': value_list[21], 'Total Cash':
                            value_list[22], 'Total Cash Per Share':
                            value_list[23], 'Total Debt': value_list[24],
                            'Current Ratio': value_list[25],
                            'Book Value Per Share': value_list[26],
                            'Cash Flow': value_list[27], 'Beta': value_list
                            [28], 'Held by Insiders': value_list[29],
                            'Held by Institutions': value_list[30],
                            'Shares Short (as of': value_list[31],
                            'Short Ratio': value_list[32],
                            'Short % of Float': value_list[33],
                            'Shares Short (prior ': value_list[34],
                            'Status': status}, index=[0])
                        df2 = df2.reindex(columns=dfSequence)
                        df = df.append(df2, ignore_index=True)
                except Exception as e:
                    print(e)
    df.to_csv('key_stats.csv')


<mask token>

<mask token>
style.use('dark_background')
<mask token>


def Key_Stats(gather=['Total Debt/Equity', 'Trailing P/E', 'Price/Sales',
    'Price/Book', 'Profit Margin', 'Operating Margin', 'Return on Assets',
    'Return on Equity', 'Revenue Per Share', 'Market Cap',
    'Enterprise Value', 'Forward P/E', 'PEG Ratio',
    'Enterprise Value/Revenue', 'Enterprise Value/EBITDA', 'Revenue',
    'Gross Profit', 'EBITDA', 'Net Income Avl to Common ', 'Diluted EPS',
    'Earnings Growth', 'Revenue Growth', 'Total Cash',
    'Total Cash Per Share', 'Total Debt', 'Current Ratio',
    'Book Value Per Share', 'Cash Flow', 'Beta', 'Held by Insiders',
    'Held by Institutions', 'Shares Short (as of', 'Short Ratio',
    'Short % of Float', 'Shares Short (prior ']):
    statspath = path + '/_KeyStats'
    stock_list = [x[0] for x in os.walk(statspath)]
    dfSequence = ['Date', 'Unix', 'Ticker', 'DE Ratio', 'Price',
        'stock_p_change', 'SP500', 'sp500_p_change', 'Difference',
        'DE Ratio', 'Trailing P/E', 'Price/Sales', 'Price/Book',
        'Profit Margin', 'Operating Margin', 'Return on Assets',
        'Return on Equity', 'Revenue Per Share', 'Market Cap',
        'Enterprise Value', 'Forward P/E', 'PEG Ratio',
        'Enterprise Value/Revenue', 'Enterprise Value/EBITDA', 'Revenue',
        'Gross Profit', 'EBITDA', 'Net Income Avl to Common ',
        'Diluted EPS', 'Earnings Growth', 'Revenue Growth', 'Total Cash',
        'Total Cash Per Share', 'Total Debt', 'Current Ratio',
        'Book Value Per Share', 'Cash Flow', 'Beta', 'Held by Insiders',
        'Held by Institutions', 'Shares Short (as of', 'Short Ratio',
        'Short % of Float', 'Shares Short (prior ', 'Status']
    df = pd.DataFrame(columns=['Date', 'Unix', 'Ticker', 'DE Ratio',
        'Price', 'stock_p_change', 'SP500', 'sp500_p_change', 'Difference',
        'DE Ratio', 'Trailing P/E', 'Price/Sales', 'Price/Book',
        'Profit Margin', 'Operating Margin', 'Return on Assets',
        'Return on Equity', 'Revenue Per Share', 'Market Cap',
        'Enterprise Value', 'Forward P/E', 'PEG Ratio',
        'Enterprise Value/Revenue', 'Enterprise Value/EBITDA', 'Revenue',
        'Gross Profit', 'EBITDA', 'Net Income Avl to Common ',
        'Diluted EPS', 'Earnings Growth', 'Revenue Growth', 'Total Cash',
        'Total Cash Per Share', 'Total Debt', 'Current Ratio',
        'Book Value Per Share', 'Cash Flow', 'Beta', 'Held by Insiders',
        'Held by Institutions', 'Shares Short (as of', 'Short Ratio',
        'Short % of Float', 'Shares Short (prior ', 'Status'])
    sp500_df = pd.DataFrame.from_csv(homepath + '/YAHOO-INDEX_GSPC.csv')
    ticker_list = []
    for each_dir in stock_list[1:]:
        each_file = os.listdir(each_dir)
        dir_split = each_dir.split('/')
        ticker = dir_split[len(dir_split) - 1]
        ticker_list.append(ticker)
        starting_stock_value = False
        starting_sp500_value = False
        if len(each_file) > 0:
            for file in each_file:
                date_stamp = datetime.strptime(file, '%Y%m%d%H%M%S.html')
                unix_time = time.mktime(date_stamp.timetuple())
                full_file_path = each_dir + '/' + file
                source = open(full_file_path, 'r').read()
                try:
                    value_list = []
                    for each_data in gather:
                        try:
                            regex = re.escape(each_data
                                ) + '.*?(\\d{1,8}\\.\\d{1,8}M?B?|N/A)%?</td>'
                            value = re.search(regex, source)
                            value = value.group(1)
                            if 'B' in value:
                                value = float(value.replace('B', '')
                                    ) * 1000000000
                            elif 'M' in value:
                                value = float(value.replace('M', '')) * 1000000
                            value_list.append(value)
                        except Exception as e:
                            value = 'N/A'
                            value_list.append(value)
                    try:
                        sp500_date = datetime.fromtimestamp(unix_time)
                        sp500_date = sp500_date.strftime('%Y-%m-%d')
                        row = sp500_df[sp500_df.index == sp500_date]
                        sp500_value = float(row['Adjusted Close'])
                    except:
                        sp500_time = unix_time
                        lenRow = 0
                        while lenRow == 0:
                            sp500_time = sp500_time - 86400
                            sp500_date = datetime.fromtimestamp(sp500_time)
                            sp500_date = sp500_date.strftime('%Y-%m-%d')
                            row = sp500_df[sp500_df.index == sp500_date]
                            lenRow = len(row)
                        sp500_value = float(row['Adjusted Close'])
                    try:
                        stock_price = float(source.split('</small><big><b>'
                            )[1].split('</b></big>')[0])
                    except Exception as e:
                        try:
                            stock_price = source.split('</small><big><b>')[1
                                ].split('</b></big>')[0]
                            stock_price = re.search('(\\d{1,8}\\.\\d{1,8})',
                                stock_price)
                            stock_price = float(stock_price.group(1))
                        except Exception as e:
                            try:
                                stock_price = source.split(
                                    '<span class="time_rtq_ticker">')[1].split(
                                    '</span>')[0]
                                stock_price = re.search('(\\d{1,8}\\.\\d{1,8})'
                                    , stock_price)
                                stock_price = float(stock_price.group(1))
                            except Exception as e:
                                print(str(e), 'a;lsdkfh', file, ticker)
                    if not starting_stock_value:
                        starting_stock_value = stock_price
                    if not starting_sp500_value:
                        starting_sp500_value = sp500_value
                    stock_p_change = (stock_price - starting_stock_value
                        ) / starting_stock_value * 100
                    sp500_p_change = (sp500_value - starting_sp500_value
                        ) / starting_sp500_value * 100
                    difference = stock_p_change - sp500_p_change
                    if difference > 0:
                        status = 'outperform'
                    else:
                        status = 'underperform'
                    if value_list.count('N/A') > 0:
                        pass
                    else:
                        df2 = pd.DataFrame({'Date': date_stamp, 'Unix':
                            unix_time, 'Ticker': ticker, 'Price':
                            stock_price, 'stock_p_change': stock_p_change,
                            'SP500': sp500_value, 'sp500_p_change':
                            sp500_p_change, 'Difference': difference,
                            'DE Ratio': value_list[0], 'Trailing P/E':
                            value_list[1], 'Price/Sales': value_list[2],
                            'Price/Book': value_list[3], 'Profit Margin':
                            value_list[4], 'Operating Margin': value_list[5
                            ], 'Return on Assets': value_list[6],
                            'Return on Equity': value_list[7],
                            'Revenue Per Share': value_list[8],
                            'Market Cap': value_list[9], 'Enterprise Value':
                            value_list[10], 'Forward P/E': value_list[11],
                            'PEG Ratio': value_list[12],
                            'Enterprise Value/Revenue': value_list[13],
                            'Enterprise Value/EBITDA': value_list[14],
                            'Revenue': value_list[15], 'Gross Profit':
                            value_list[16], 'EBITDA': value_list[17],
                            'Net Income Avl to Common ': value_list[18],
                            'Diluted EPS': value_list[19],
                            'Earnings Growth': value_list[20],
                            'Revenue Growth': value_list[21], 'Total Cash':
                            value_list[22], 'Total Cash Per Share':
                            value_list[23], 'Total Debt': value_list[24],
                            'Current Ratio': value_list[25],
                            'Book Value Per Share': value_list[26],
                            'Cash Flow': value_list[27], 'Beta': value_list
                            [28], 'Held by Insiders': value_list[29],
                            'Held by Institutions': value_list[30],
                            'Shares Short (as of': value_list[31],
                            'Short Ratio': value_list[32],
                            'Short % of Float': value_list[33],
                            'Shares Short (prior ': value_list[34],
                            'Status': status}, index=[0])
                        df2 = df2.reindex(columns=dfSequence)
                        df = df.append(df2, ignore_index=True)
                except Exception as e:
                    print(e)
    df.to_csv('key_stats.csv')


Key_Stats()

<mask token>
style.use('dark_background')
homepath = (
    '/Volumes/SD64/Dropbox/200.CaptainResearch/Python_MachineLearning/PythonML'
    )
path = homepath + '/intraQuarter'


def Key_Stats(gather=['Total Debt/Equity', 'Trailing P/E', 'Price/Sales',
    'Price/Book', 'Profit Margin', 'Operating Margin', 'Return on Assets',
    'Return on Equity', 'Revenue Per Share', 'Market Cap',
    'Enterprise Value', 'Forward P/E', 'PEG Ratio',
    'Enterprise Value/Revenue', 'Enterprise Value/EBITDA', 'Revenue',
    'Gross Profit', 'EBITDA', 'Net Income Avl to Common ', 'Diluted EPS',
    'Earnings Growth', 'Revenue Growth', 'Total Cash',
    'Total Cash Per Share', 'Total Debt', 'Current Ratio',
    'Book Value Per Share', 'Cash Flow', 'Beta', 'Held by Insiders',
    'Held by Institutions', 'Shares Short (as of', 'Short Ratio',
    'Short % of Float', 'Shares Short (prior ']):
    statspath = path + '/_KeyStats'
    stock_list = [x[0] for x in os.walk(statspath)]
    dfSequence = ['Date', 'Unix', 'Ticker', 'DE Ratio', 'Price',
        'stock_p_change', 'SP500', 'sp500_p_change', 'Difference',
        'DE Ratio', 'Trailing P/E', 'Price/Sales', 'Price/Book',
        'Profit Margin', 'Operating Margin', 'Return on Assets',
        'Return on Equity', 'Revenue Per Share', 'Market Cap',
        'Enterprise Value', 'Forward P/E', 'PEG Ratio',
        'Enterprise Value/Revenue', 'Enterprise Value/EBITDA', 'Revenue',
        'Gross Profit', 'EBITDA', 'Net Income Avl to Common ',
        'Diluted EPS', 'Earnings Growth', 'Revenue Growth', 'Total Cash',
        'Total Cash Per Share', 'Total Debt', 'Current Ratio',
        'Book Value Per Share', 'Cash Flow', 'Beta', 'Held by Insiders',
        'Held by Institutions', 'Shares Short (as of', 'Short Ratio',
        'Short % of Float', 'Shares Short (prior ', 'Status']
    df = pd.DataFrame(columns=['Date', 'Unix', 'Ticker', 'DE Ratio',
        'Price', 'stock_p_change', 'SP500', 'sp500_p_change', 'Difference',
        'DE Ratio', 'Trailing P/E', 'Price/Sales', 'Price/Book',
        'Profit Margin', 'Operating Margin', 'Return on Assets',
        'Return on Equity', 'Revenue Per Share', 'Market Cap',
        'Enterprise Value', 'Forward P/E', 'PEG Ratio',
        'Enterprise Value/Revenue', 'Enterprise Value/EBITDA', 'Revenue',
        'Gross Profit', 'EBITDA', 'Net Income Avl to Common ',
        'Diluted EPS', 'Earnings Growth', 'Revenue Growth', 'Total Cash',
        'Total Cash Per Share', 'Total Debt', 'Current Ratio',
        'Book Value Per Share', 'Cash Flow', 'Beta', 'Held by Insiders',
        'Held by Institutions', 'Shares Short (as of', 'Short Ratio',
        'Short % of Float', 'Shares Short (prior ', 'Status'])
    sp500_df = pd.DataFrame.from_csv(homepath + '/YAHOO-INDEX_GSPC.csv')
    ticker_list = []
    for each_dir in stock_list[1:]:
        each_file = os.listdir(each_dir)
        dir_split = each_dir.split('/')
        ticker = dir_split[len(dir_split) - 1]
        ticker_list.append(ticker)
        starting_stock_value = False
        starting_sp500_value = False
        if len(each_file) > 0:
            for file in each_file:
                date_stamp = datetime.strptime(file, '%Y%m%d%H%M%S.html')
                unix_time = time.mktime(date_stamp.timetuple())
                full_file_path = each_dir + '/' + file
                source = open(full_file_path, 'r').read()
                try:
                    value_list = []
                    for each_data in gather:
                        try:
                            regex = re.escape(each_data
                                ) + '.*?(\\d{1,8}\\.\\d{1,8}M?B?|N/A)%?</td>'
                            value = re.search(regex, source)
                            value = value.group(1)
                            if 'B' in value:
                                value = float(value.replace('B', '')
                                    ) * 1000000000
                            elif 'M' in value:
                                value = float(value.replace('M', '')) * 1000000
                            value_list.append(value)
                        except Exception as e:
                            value = 'N/A'
                            value_list.append(value)
                    try:
                        sp500_date = datetime.fromtimestamp(unix_time)
                        sp500_date = sp500_date.strftime('%Y-%m-%d')
                        row = sp500_df[sp500_df.index == sp500_date]
                        sp500_value = float(row['Adjusted Close'])
                    except:
                        sp500_time = unix_time
                        lenRow = 0
                        while lenRow == 0:
                            sp500_time = sp500_time - 86400
                            sp500_date = datetime.fromtimestamp(sp500_time)
                            sp500_date = sp500_date.strftime('%Y-%m-%d')
                            row = sp500_df[sp500_df.index == sp500_date]
                            lenRow = len(row)
                        sp500_value = float(row['Adjusted Close'])
                    try:
                        stock_price = float(source.split('</small><big><b>'
                            )[1].split('</b></big>')[0])
                    except Exception as e:
                        try:
                            stock_price = source.split('</small><big><b>')[1
                                ].split('</b></big>')[0]
                            stock_price = re.search('(\\d{1,8}\\.\\d{1,8})',
                                stock_price)
                            stock_price = float(stock_price.group(1))
                        except Exception as e:
                            try:
                                stock_price = source.split(
                                    '<span class="time_rtq_ticker">')[1].split(
                                    '</span>')[0]
                                stock_price = re.search('(\\d{1,8}\\.\\d{1,8})'
                                    , stock_price)
                                stock_price = float(stock_price.group(1))
                            except Exception as e:
                                print(str(e), 'a;lsdkfh', file, ticker)
                    if not starting_stock_value:
                        starting_stock_value = stock_price
                    if not starting_sp500_value:
                        starting_sp500_value = sp500_value
                    stock_p_change = (stock_price - starting_stock_value
                        ) / starting_stock_value * 100
                    sp500_p_change = (sp500_value - starting_sp500_value
                        ) / starting_sp500_value * 100
                    difference = stock_p_change - sp500_p_change
                    if difference > 0:
                        status = 'outperform'
                    else:
                        status = 'underperform'
                    if value_list.count('N/A') > 0:
                        pass
                    else:
                        df2 = pd.DataFrame({'Date': date_stamp, 'Unix':
                            unix_time, 'Ticker': ticker, 'Price':
                            stock_price, 'stock_p_change': stock_p_change,
                            'SP500': sp500_value, 'sp500_p_change':
                            sp500_p_change, 'Difference': difference,
                            'DE Ratio': value_list[0], 'Trailing P/E':
                            value_list[1], 'Price/Sales': value_list[2],
                            'Price/Book': value_list[3], 'Profit Margin':
                            value_list[4], 'Operating Margin': value_list[5
                            ], 'Return on Assets': value_list[6],
                            'Return on Equity': value_list[7],
                            'Revenue Per Share': value_list[8],
                            'Market Cap': value_list[9], 'Enterprise Value':
                            value_list[10], 'Forward P/E': value_list[11],
                            'PEG Ratio': value_list[12],
                            'Enterprise Value/Revenue': value_list[13],
                            'Enterprise Value/EBITDA': value_list[14],
                            'Revenue': value_list[15], 'Gross Profit':
                            value_list[16], 'EBITDA': value_list[17],
                            'Net Income Avl to Common ': value_list[18],
                            'Diluted EPS': value_list[19],
                            'Earnings Growth': value_list[20],
                            'Revenue Growth': value_list[21], 'Total Cash':
                            value_list[22], 'Total Cash Per Share':
                            value_list[23], 'Total Debt': value_list[24],
                            'Current Ratio': value_list[25],
                            'Book Value Per Share': value_list[26],
                            'Cash Flow': value_list[27], 'Beta': value_list
                            [28], 'Held by Insiders': value_list[29],
                            'Held by Institutions': value_list[30],
                            'Shares Short (as of': value_list[31],
                            'Short Ratio': value_list[32],
                            'Short % of Float': value_list[33],
                            'Shares Short (prior ': value_list[34],
                            'Status': status}, index=[0])
                        df2 = df2.reindex(columns=dfSequence)
                        df = df.append(df2, ignore_index=True)
                except Exception as e:
                    print(e)
    df.to_csv('key_stats.csv')


Key_Stats()

import pandas as pd
import os
import time
from datetime import datetime
import re
from matplotlib import style
style.use('dark_background')
homepath = (
    '/Volumes/SD64/Dropbox/200.CaptainResearch/Python_MachineLearning/PythonML'
    )
path = homepath + '/intraQuarter'


def Key_Stats(gather=['Total Debt/Equity', 'Trailing P/E', 'Price/Sales',
    'Price/Book', 'Profit Margin', 'Operating Margin', 'Return on Assets',
    'Return on Equity', 'Revenue Per Share', 'Market Cap',
    'Enterprise Value', 'Forward P/E', 'PEG Ratio',
    'Enterprise Value/Revenue', 'Enterprise Value/EBITDA', 'Revenue',
    'Gross Profit', 'EBITDA', 'Net Income Avl to Common ', 'Diluted EPS',
    'Earnings Growth', 'Revenue Growth', 'Total Cash',
    'Total Cash Per Share', 'Total Debt', 'Current Ratio',
    'Book Value Per Share', 'Cash Flow', 'Beta', 'Held by Insiders',
    'Held by Institutions', 'Shares Short (as of', 'Short Ratio',
    'Short % of Float', 'Shares Short (prior ']):
    statspath = path + '/_KeyStats'
    stock_list = [x[0] for x in os.walk(statspath)]
    dfSequence = ['Date', 'Unix', 'Ticker', 'DE Ratio', 'Price',
        'stock_p_change', 'SP500', 'sp500_p_change', 'Difference',
        'DE Ratio', 'Trailing P/E', 'Price/Sales', 'Price/Book',
        'Profit Margin', 'Operating Margin', 'Return on Assets',
        'Return on Equity', 'Revenue Per Share', 'Market Cap',
        'Enterprise Value', 'Forward P/E', 'PEG Ratio',
        'Enterprise Value/Revenue', 'Enterprise Value/EBITDA', 'Revenue',
        'Gross Profit', 'EBITDA', 'Net Income Avl to Common ',
        'Diluted EPS', 'Earnings Growth', 'Revenue Growth', 'Total Cash',
        'Total Cash Per Share', 'Total Debt', 'Current Ratio',
        'Book Value Per Share', 'Cash Flow', 'Beta', 'Held by Insiders',
        'Held by Institutions', 'Shares Short (as of', 'Short Ratio',
        'Short % of Float', 'Shares Short (prior ', 'Status']
    df = pd.DataFrame(columns=['Date', 'Unix', 'Ticker', 'DE Ratio',
        'Price', 'stock_p_change', 'SP500', 'sp500_p_change', 'Difference',
        'DE Ratio', 'Trailing P/E', 'Price/Sales', 'Price/Book',
        'Profit Margin', 'Operating Margin', 'Return on Assets',
        'Return on Equity', 'Revenue Per Share', 'Market Cap',
        'Enterprise Value', 'Forward P/E', 'PEG Ratio',
        'Enterprise Value/Revenue', 'Enterprise Value/EBITDA', 'Revenue',
        'Gross Profit', 'EBITDA', 'Net Income Avl to Common ',
        'Diluted EPS', 'Earnings Growth', 'Revenue Growth', 'Total Cash',
        'Total Cash Per Share', 'Total Debt', 'Current Ratio',
        'Book Value Per Share', 'Cash Flow', 'Beta', 'Held by Insiders',
        'Held by Institutions', 'Shares Short (as of', 'Short Ratio',
        'Short % of Float', 'Shares Short (prior ', 'Status'])
    sp500_df = pd.DataFrame.from_csv(homepath + '/YAHOO-INDEX_GSPC.csv')
    ticker_list = []
    for each_dir in stock_list[1:]:
        each_file = os.listdir(each_dir)
        dir_split = each_dir.split('/')
        ticker = dir_split[len(dir_split) - 1]
        ticker_list.append(ticker)
        starting_stock_value = False
        starting_sp500_value = False
        if len(each_file) > 0:
            for file in each_file:
                date_stamp = datetime.strptime(file, '%Y%m%d%H%M%S.html')
                unix_time = time.mktime(date_stamp.timetuple())
                full_file_path = each_dir + '/' + file
                source = open(full_file_path, 'r').read()
                try:
                    value_list = []
                    for each_data in gather:
                        try:
                            regex = re.escape(each_data
                                ) + '.*?(\\d{1,8}\\.\\d{1,8}M?B?|N/A)%?</td>'
                            value = re.search(regex, source)
                            value = value.group(1)
                            if 'B' in value:
                                value = float(value.replace('B', '')
                                    ) * 1000000000
                            elif 'M' in value:
                                value = float(value.replace('M', '')) * 1000000
                            value_list.append(value)
                        except Exception as e:
                            value = 'N/A'
                            value_list.append(value)
                    try:
                        sp500_date = datetime.fromtimestamp(unix_time)
                        sp500_date = sp500_date.strftime('%Y-%m-%d')
                        row = sp500_df[sp500_df.index == sp500_date]
                        sp500_value = float(row['Adjusted Close'])
                    except:
                        sp500_time = unix_time
                        lenRow = 0
                        while lenRow == 0:
                            sp500_time = sp500_time - 86400
                            sp500_date = datetime.fromtimestamp(sp500_time)
                            sp500_date = sp500_date.strftime('%Y-%m-%d')
                            row = sp500_df[sp500_df.index == sp500_date]
                            lenRow = len(row)
                        sp500_value = float(row['Adjusted Close'])
                    try:
                        stock_price = float(source.split('</small><big><b>'
                            )[1].split('</b></big>')[0])
                    except Exception as e:
                        try:
                            stock_price = source.split('</small><big><b>')[1
                                ].split('</b></big>')[0]
                            stock_price = re.search('(\\d{1,8}\\.\\d{1,8})',
                                stock_price)
                            stock_price = float(stock_price.group(1))
                        except Exception as e:
                            try:
                                stock_price = source.split(
                                    '<span class="time_rtq_ticker">')[1].split(
                                    '</span>')[0]
                                stock_price = re.search('(\\d{1,8}\\.\\d{1,8})'
                                    , stock_price)
                                stock_price = float(stock_price.group(1))
                            except Exception as e:
                                print(str(e), 'a;lsdkfh', file, ticker)
                    if not starting_stock_value:
                        starting_stock_value = stock_price
                    if not starting_sp500_value:
                        starting_sp500_value = sp500_value
                    stock_p_change = (stock_price - starting_stock_value
                        ) / starting_stock_value * 100
                    sp500_p_change = (sp500_value - starting_sp500_value
                        ) / starting_sp500_value * 100
                    difference = stock_p_change - sp500_p_change
                    if difference > 0:
                        status = 'outperform'
                    else:
                        status = 'underperform'
                    if value_list.count('N/A') > 0:
                        pass
                    else:
                        df2 = pd.DataFrame({'Date': date_stamp, 'Unix':
                            unix_time, 'Ticker': ticker, 'Price':
                            stock_price, 'stock_p_change': stock_p_change,
                            'SP500': sp500_value, 'sp500_p_change':
                            sp500_p_change, 'Difference': difference,
                            'DE Ratio': value_list[0], 'Trailing P/E':
                            value_list[1], 'Price/Sales': value_list[2],
                            'Price/Book': value_list[3], 'Profit Margin':
                            value_list[4], 'Operating Margin': value_list[5
                            ], 'Return on Assets': value_list[6],
                            'Return on Equity': value_list[7],
                            'Revenue Per Share': value_list[8],
                            'Market Cap': value_list[9], 'Enterprise Value':
                            value_list[10], 'Forward P/E': value_list[11],
                            'PEG Ratio': value_list[12],
                            'Enterprise Value/Revenue': value_list[13],
                            'Enterprise Value/EBITDA': value_list[14],
                            'Revenue': value_list[15], 'Gross Profit':
                            value_list[16], 'EBITDA': value_list[17],
                            'Net Income Avl to Common ': value_list[18],
                            'Diluted EPS': value_list[19],
                            'Earnings Growth': value_list[20],
                            'Revenue Growth': value_list[21], 'Total Cash':
                            value_list[22], 'Total Cash Per Share':
                            value_list[23], 'Total Debt': value_list[24],
                            'Current Ratio': value_list[25],
                            'Book Value Per Share': value_list[26],
                            'Cash Flow': value_list[27], 'Beta': value_list
                            [28], 'Held by Insiders': value_list[29],
                            'Held by Institutions': value_list[30],
                            'Shares Short (as of': value_list[31],
                            'Short Ratio': value_list[32],
                            'Short % of Float': value_list[33],
                            'Shares Short (prior ': value_list[34],
                            'Status': status}, index=[0])
                        df2 = df2.reindex(columns=dfSequence)
                        df = df.append(df2, ignore_index=True)
                except Exception as e:
                    print(e)
    df.to_csv('key_stats.csv')


Key_Stats()

# -*-coding: utf-8 -*-
# ref scikit-learn.org/stable/tutorial/machine_learning_map/index.html
# ref https://pythonprogramming.net/getting-data-sp-500-index-value-comparison
#           /?completed=/using-pandas-structure-process-data/
# 1. data
# 2. labeling
# 3. choose right machine learning algorithm

# stock fundemental
# Parsing data
import pandas as pd
import os   # interact with directories
import time
from datetime import datetime
# from time import mktime

# import matplotlib
# import matplotlib.pyplot as plt

import re
# import urllib

from matplotlib import style
style.use('dark_background')


# homepath = "/Users/jihwanpark/PythonML"
homepath = "/Volumes/SD64/Dropbox/200.CaptainResearch/Python_MachineLearning/PythonML"
# homepath = "/Users/ehmp/Dropbox/200.CaptainResearch/Python_MachineLearning/PythonML"

path = homepath+"/intraQuarter"


def Key_Stats(gather=["Total Debt/Equity",
                      'Trailing P/E',
                      'Price/Sales',
                      'Price/Book',
                      'Profit Margin',
                      'Operating Margin',
                      'Return on Assets',
                      'Return on Equity',
                      'Revenue Per Share',
                      'Market Cap',
                      'Enterprise Value',
                      'Forward P/E',
                      'PEG Ratio',
                      'Enterprise Value/Revenue',
                      'Enterprise Value/EBITDA',
                      'Revenue',
                      'Gross Profit',
                      'EBITDA',
                      'Net Income Avl to Common ',
                      'Diluted EPS',
                      'Earnings Growth',
                      'Revenue Growth',
                      'Total Cash',
                      'Total Cash Per Share',
                      'Total Debt',
                      'Current Ratio',
                      'Book Value Per Share',
                      'Cash Flow',
                      'Beta',
                      'Held by Insiders',
                      'Held by Institutions',
                      'Shares Short (as of',
                      'Short Ratio',
                      'Short % of Float',
                      'Shares Short (prior ']):

    statspath = path+'/_KeyStats'   # stats directory
    stock_list = [x[0] for x in os.walk(statspath)]

    dfSequence = ['Date',
                               'Unix',
                               'Ticker',
                               'DE Ratio',
                               'Price',
                               'stock_p_change',
                               'SP500',
                               'sp500_p_change',
                               'Difference',
                               'DE Ratio',
                               'Trailing P/E',
                               'Price/Sales',
                               'Price/Book',
                               'Profit Margin',
                               'Operating Margin',
                               'Return on Assets',
                               'Return on Equity',
                               'Revenue Per Share',
                               'Market Cap',
                               'Enterprise Value',
                               'Forward P/E',
                               'PEG Ratio',
                               'Enterprise Value/Revenue',
                               'Enterprise Value/EBITDA',
                               'Revenue',
                               'Gross Profit',
                               'EBITDA',
                               'Net Income Avl to Common ',
                               'Diluted EPS',
                               'Earnings Growth',
                               'Revenue Growth',
                               'Total Cash',
                               'Total Cash Per Share',
                               'Total Debt',
                               'Current Ratio',
                               'Book Value Per Share',
                               'Cash Flow',
                               'Beta',
                               'Held by Insiders',
                               'Held by Institutions',
                               'Shares Short (as of',
                               'Short Ratio',
                               'Short % of Float',
                               'Shares Short (prior ',
                               'Status']

    df = pd.DataFrame(columns=['Date',
                               'Unix',
                               'Ticker',
                               'DE Ratio',
                               'Price',
                               'stock_p_change',
                               'SP500',
                               'sp500_p_change',
                               'Difference',
                               'DE Ratio',
                               'Trailing P/E',
                               'Price/Sales',
                               'Price/Book',
                               'Profit Margin',
                               'Operating Margin',
                               'Return on Assets',
                               'Return on Equity',
                               'Revenue Per Share',
                               'Market Cap',
                               'Enterprise Value',
                               'Forward P/E',
                               'PEG Ratio',
                               'Enterprise Value/Revenue',
                               'Enterprise Value/EBITDA',
                               'Revenue',
                               'Gross Profit',
                               'EBITDA',
                               'Net Income Avl to Common ',
                               'Diluted EPS',
                               'Earnings Growth',
                               'Revenue Growth',
                               'Total Cash',
                               'Total Cash Per Share',
                               'Total Debt',
                               'Current Ratio',
                               'Book Value Per Share',
                               'Cash Flow',
                               'Beta',
                               'Held by Insiders',
                               'Held by Institutions',
                               'Shares Short (as of',
                               'Short Ratio',
                               'Short % of Float',
                               'Shares Short (prior ',
                               'Status'])

    sp500_df = pd.DataFrame.from_csv(homepath+"/YAHOO-INDEX_GSPC.csv")

    ticker_list = []

    for each_dir in stock_list[1:]:  # 25 is test element number
        each_file = os.listdir(each_dir)

        dir_split = each_dir.split("/")
        ticker = dir_split[len(dir_split)-1]
        ticker_list.append(ticker)
        starting_stock_value = False
        starting_sp500_value = False

        if len(each_file) > 0:

            for file in each_file:

                date_stamp = datetime.strptime(file, '%Y%m%d%H%M%S.html')
                unix_time = time.mktime(date_stamp.timetuple())
                full_file_path = each_dir+'/'+file
                source = open(full_file_path, 'r').read()
                try:
                    value_list = []

                    for each_data in gather:
                        try:
                            regex = re.escape(each_data) + r'.*?(\d{1,8}\.\d{1,8}M?B?|N/A)%?</td>'
                            value = re.search(regex, source)
                            value = (value.group(1))

                            if "B" in value:
                                value = float(value.replace("B", ''))*1000000000
                            elif "M" in value:
                                value = float(value.replace("M", ''))*1000000

                            value_list.append(value)

                        except Exception as e:
                            value = "N/A"
                            value_list.append(value)
                    # print(value_list)
                    try:
                        sp500_date = datetime.fromtimestamp(unix_time)
                        sp500_date = sp500_date.strftime('%Y-%m-%d')
                        row = sp500_df[(sp500_df.index == sp500_date)]
                        sp500_value = float(row["Adjusted Close"])
                    except:
                        sp500_time = unix_time
                        lenRow = 0
                        while lenRow == 0:
                            sp500_time = sp500_time - 86400
                            sp500_date = datetime.fromtimestamp(sp500_time)
                            sp500_date = sp500_date.strftime('%Y-%m-%d')
                            row = sp500_df[(sp500_df.index == sp500_date)]
                            lenRow = len(row)
                        sp500_value = float(row["Adjusted Close"])

                    # get stock price
                    try:
                        stock_price = float(source.split('</small><big><b>')[1].split('</b></big>')[0])
                    except Exception as e:
                        #    <span id="yfs_l10_afl">43.27</span>
                        try:
                            stock_price = (source.split('</small><big><b>')[1].split('</b></big>')[0])
                            stock_price = re.search(r'(\d{1,8}\.\d{1,8})', stock_price)
                            stock_price = float(stock_price.group(1))

                            # print(stock_price)
                        except Exception as e:
                            try:
                                stock_price = (source.split('<span class="time_rtq_ticker">')[1].split('</span>')[0])
                                stock_price = re.search(r'(\d{1,8}\.\d{1,8})', stock_price)
                                stock_price = float(stock_price.group(1))
                            except Exception as e:
                                print(str(e), 'a;lsdkfh', file, ticker)

                            # print('Latest:',stock_price)

                            # print('stock price',str(e),ticker,file)
                            # time.sleep(15)

                    # print("stock_price:",stock_price,"ticker:", ticker)
                    if not starting_stock_value:
                        starting_stock_value = stock_price
                    if not starting_sp500_value:
                        starting_sp500_value = sp500_value

                    stock_p_change = ((stock_price - starting_stock_value)
                                      / starting_stock_value) * 100
                    sp500_p_change = ((sp500_value - starting_sp500_value)
                                      / starting_sp500_value) * 100

                    difference = stock_p_change - sp500_p_change
                    if difference > 0:
                        status = "outperform"
                    else:
                        status = "underperform"

                    if value_list.count("N/A") > 0:
                        pass
                    else:
                        # df = df.append({
                        df2 = pd.DataFrame({
                           'Date': date_stamp,
                           'Unix': unix_time,
                           'Ticker': ticker,
                           'Price': stock_price,
                           'stock_p_change': stock_p_change,
                           'SP500': sp500_value,
                           'sp500_p_change': sp500_p_change,
                           'Difference': difference,
                           'DE Ratio': value_list[0],
                           'Trailing P/E': value_list[1],
                           'Price/Sales': value_list[2],
                           'Price/Book': value_list[3],
                           'Profit Margin': value_list[4],
                           'Operating Margin': value_list[5],
                           'Return on Assets': value_list[6],
                           'Return on Equity': value_list[7],
                           'Revenue Per Share': value_list[8],
                           'Market Cap': value_list[9],
                           'Enterprise Value': value_list[10],
                           'Forward P/E': value_list[11],
                           'PEG Ratio': value_list[12],
                           'Enterprise Value/Revenue': value_list[13],
                           'Enterprise Value/EBITDA': value_list[14],
                           'Revenue': value_list[15],
                           'Gross Profit': value_list[16],
                           'EBITDA': value_list[17],
                           'Net Income Avl to Common ': value_list[18],
                           'Diluted EPS': value_list[19],
                           'Earnings Growth': value_list[20],
                           'Revenue Growth': value_list[21],
                           'Total Cash': value_list[22],
                           'Total Cash Per Share': value_list[23],
                           'Total Debt': value_list[24],
                           'Current Ratio': value_list[25],
                           'Book Value Per Share': value_list[26],
                           'Cash Flow': value_list[27],
                           'Beta': value_list[28],
                           'Held by Insiders': value_list[29],
                           'Held by Institutions': value_list[30],
                           'Shares Short (as of': value_list[31],
                           'Short Ratio': value_list[32],
                           'Short % of Float': value_list[33],
                           'Shares Short (prior ': value_list[34],
                           'Status': status}, index=[0])
                        df2 = df2.reindex(columns=dfSequence)
                        df = df.append(df2, ignore_index=True)

                except Exception as e:
                    print(e)
                    # pass

    df.to_csv("key_stats.csv")


Key_Stats()
