import logging
from datetime import datetime

from Format3.format_3B.format3_html import Format3html
from config.config import CONFIG
from bs4 import BeautifulSoup

from Format3.format_3B.format3b_parser import Format3Parser


class Format3Pipeline:

    # def __init__(self):
        # logging.config.fileConfig('loggerconfig.conf')
        # self.logger = logging.getLogger('MainLogger')
        # fh = logging.FileHandler(CONFIG["Path"]["log_path"]+'{:%Y-%m-%d}.log'.format(datetime.now()))
        # formatter = logging.Formatter('%(asctime)s | %(levelname)-8s | %(lineno)04d | %(message)s')
        # fh.setFormatter(formatter)
        # self.logger.addHandler(fh)

    def processFormat3B(self,accession_no,documentId):
        try:
            content=Format3Parser.read_file(accession_no)
            clean_html=Format3html.cleanMe(content)
            clean_html=Format3Parser.segparsing(clean_html)
            # clean_html=clean_html.replace("<div>","")
            # clean_html=clean_html.replace("</div>","")
            # print(clean_html,"\n\n\n")
            tables=Format3Parser.returnTables(clean_html)
            print("type of tables elt is",type(tables))
            Format3Parser.df_toexcel(tables,accession_no,documentId,filePath=0)
            #Format3Parser.df_tosql(df,accession_no,documentId)
        except Exception as e:
            print("Exception in parsing for accession no{} is {}: ".format(accession_no,e))

if __name__=="__main__":

    accession_list=["0001387131-18-004331",
    "0000928816-18-001652",
    "0000928816-18-001654","0000051931-18-000823",
    "0000051931-18-000818","0000894189-18-004917",
"0000898432-18-000966",
"0000898432-18-000968",
"0000898432-18-000970",
"0000898432-18-000971",
"0000898432-18-000972",
"0001144204-18-043917",
"0001144204-18-046934",
"0001144204-18-046936",
"0001144204-18-046940",
"0001144204-18-046943",
"0001193125-18-263173",
"0001193125-18-263198",
"0001193125-18-263271",
"0001615774-18-007808","0000051931-18-000805",
"0000051931-18-000811",
"0000051931-18-000812","0000928816-18-001652",
"0000928816-18-001654",
"0001387131-18-004331","0000928816-19-001249",
"0000928816-19-001253",
"0000928816-19-001255","0000928816-19-001271",
"0000928816-19-001274"]

new_acc_list=["0001144204-19-037565",
"0000928816-19-001314",
"0000928816-19-001294",
"0000928816-19-001286",
"0000928816-19-001280",
"0000928816-19-001274",
"0000928816-19-001271",
"0000928816-19-001268",
"0000928816-19-001265",
"0000928816-19-001261",
"0000928816-19-001258",
"0000928816-19-001255",
"0000928816-19-001253",
"0000928816-19-001249",
"0000928816-19-001248",
"0000928816-19-001245",
"0000928816-19-001243",
"0000898432-19-001139",
"0000898432-19-001138",
"0000898432-19-001137",
"0000898432-19-001134",
"0000898432-19-001133",
"0000898432-19-001132",
"0000898432-19-001130",
"0001193125-19-229467",
"0001387131-19-006459",
"0001493152-19-010998",
"0001615774-19-011383"
]

new_lot_12=["0000051931-18-000813",
"0000051931-18-000814",
"0000051931-18-000817",
"0000051931-18-000819",
"0000051931-18-000820",
"0000051931-18-000821",
"0000051931-18-000822",
"0000051931-18-000824",
"0000051931-18-000825",
"0000051931-18-000829",
"0000051931-18-000831",
"0000051931-18-000832",
"0000051931-18-000836",
"0000051931-18-000837",
"0000051931-18-000838",
"0000051931-18-000840",
"0000051931-18-000847",
"0000894189-18-004917",
"0000928816-18-001656",
"0000928816-18-001658",
"0000928816-18-001671",
"0000928816-18-001672",
"0000928816-18-001673",
"0000928816-18-001674",
"0000928816-18-001675",
"0000928816-18-001676",
"0000928816-18-001677",
"0000928816-18-001678",
"0000928816-18-001711",
"0000928816-18-001714",
"0000928816-18-001718",
"0000928816-18-001720",
"0000928816-18-001731",
"0000928816-18-001734",
"0000928816-18-001738",
"0000928816-18-001739",
"0001193125-18-252813",
"0001615774-18-007808"
]

New=["0000928816-19-001258",
"0000928816-19-001261",
"0000928816-19-001265",
"0000928816-19-001268"]

New_2312=[
"0000928816-18-001652",
"0000928816-18-001654",
"0001387131-18-004331",
"0000928816-19-001268",
"0000928816-19-001271",
"0000928816-19-001274",
"0000051931-18-000805",
"0000051931-18-000811",
"0000051931-18-000812",
"0000051931-18-000813",
"0000051931-18-000814",
"0000051931-18-000817",
"0000051931-18-000818",
"0000051931-18-000819",
"0000051931-18-000820",
"0001387131-18-004331",
"0000051931-18-000812",
"0000051931-18-000818",
"0000051931-18-000819",
"0000928816-18-001652",
"0000928816-18-001654",
"0000928816-18-001656",
"0000928816-18-001658",
"0000051931-18-000822",
"0001193125-18-252813",
"0000051931-18-000823",
"0000051931-18-000811",
"0000928816-18-001674",
"0000898432-18-000970",
"0000898432-18-000971"
]

Newlot_26122019=["0000928816-19-001249",
"0000928816-19-001253",
"0000928816-19-001255",
"0000928816-19-001258",
"0000928816-19-001261",
"0000928816-19-001265",
"0000928816-19-001268",
"0000928816-19-001271",
"0000928816-19-001274",
"0000051931-18-000805",
"0000051931-18-000811",
"0000051931-18-000812",
"0000051931-18-000813",
"0000051931-18-000814",
"0000051931-18-000817",
"0000051931-18-000818",
"0000051931-18-000819",
"0000051931-18-000820",
"0000051931-18-000821",
"0000051931-18-000822",
"0000051931-18-000823",
"0000051931-18-000824",
"0000051931-18-000825",
"0000051931-18-000829",
"0000051931-18-000831",
"0000051931-18-000832",
"0000051931-18-000836",
"0000051931-18-000837",
"0000051931-18-000838",
"0000051931-18-000840",
"0000051931-18-000847",
"0000894189-18-004917",
"0000898432-18-000966",
"0000898432-18-000968",
"0000898432-18-000970",
"0000898432-18-000971",
"0000898432-18-000972",
"0000928816-18-001652",
"0000928816-18-001654",
"0000928816-18-001656",
"0000928816-18-001658",
"0000928816-18-001671",
"0000928816-18-001672",
"0000928816-18-001673",
"0000928816-18-001674",
"0000928816-18-001675",
"0000928816-18-001676",
"0000928816-18-001677",
"0000928816-18-001678",
"0000928816-18-001711",
"0000928816-18-001714",
"0000928816-18-001718",
"0000928816-18-001720",
"0000928816-18-001731",
"0000928816-18-001734",
"0000928816-18-001738",
"0000928816-18-001739",
"0001144204-18-043917",
"0001193125-18-252813",
"0001387131-18-004331",
"0001615774-18-007808",
"0001144204-18-046934",
"0001144204-18-046936",
"0001144204-18-046940",
"0001144204-18-046943",
"0001193125-18-263173",
"0001193125-18-263198",
"0001193125-18-263271",
"0000898432-19-001130",
"0000898432-19-001133",
"0000898432-19-001134",
"0000898432-19-001137",
"0000898432-19-001138",
"0000928816-19-001243",
"0000928816-19-001245",
"0000928816-19-001248",
"0000928816-19-001280",
"0000928816-19-001286",
"0000928816-19-001294",
"0000928816-19-001314",
"0001144204-19-037565",
"0001193125-19-229467",
"0001493152-19-010998"]


ticker_missing=['0001387131-18-004331',
'0000051931-18-000812',
'0000051931-18-000818',
'0000051931-18-000819',
'0000928816-18-001652',
'0000928816-18-001654',
'0000928816-18-001656',
'0000928816-18-001658'
]

new_mismatch=['0000051931-18-000822',
              '0001193125-18-252813',
              '0000051931-18-000811',
              '0000051931-18-000805',
              '0000928816-19-001268',
              '0000928816-19-001271',
              '0000928816-19-001274',
              '0000051931-18-000805',
              '0000051931-18-000811',
              '0000051931-18-000812',
              '0000051931-18-000813',
              '0000051931-18-000814',
              '0000051931-18-000817',
              '0000051931-18-000818',
              '0000051931-18-000819',
              '0000051931-18-000820',
              '0000928816-18-001652',
              '0000928816-18-001654',
              '0001387131-18-004331',
              ]
f = Format3Pipeline()

# for accession_no in new_mismatch:
#     try:
#         f = Format3Pipeline()
#         f.processFormat3B(accession_no,'0')
#     except:
#         print("error for", accession_no)

f.processFormat3B("0000928816-19-001271",'0')
# f.processFormat3B("0000051931-18-000805",'0')
# # # f.processFormat3B("0000928816-18-001654",'0')
# f.processFormat3B("0001387131-18-004331",'0')
76003bf735a563c0b7c4ee1707304db959dca697
<mask token>


class Format3Pipeline:

    def processFormat3B(self, accession_no, documentId):
        try:
            content = Format3Parser.read_file(accession_no)
            clean_html = Format3html.cleanMe(content)
            clean_html = Format3Parser.segparsing(clean_html)
            tables = Format3Parser.returnTables(clean_html)
            print('type of tables elt is', type(tables))
            Format3Parser.df_toexcel(tables, accession_no, documentId,
                filePath=0)
        except Exception as e:
            print('Exception in parsing for accession no{} is {}: '.format(
                accession_no, e))


<mask token>

<mask token>


class Format3Pipeline:

    def processFormat3B(self, accession_no, documentId):
        try:
            content = Format3Parser.read_file(accession_no)
            clean_html = Format3html.cleanMe(content)
            clean_html = Format3Parser.segparsing(clean_html)
            tables = Format3Parser.returnTables(clean_html)
            print('type of tables elt is', type(tables))
            Format3Parser.df_toexcel(tables, accession_no, documentId,
                filePath=0)
        except Exception as e:
            print('Exception in parsing for accession no{} is {}: '.format(
                accession_no, e))


if __name__ == '__main__':
    accession_list = ['0001387131-18-004331', '0000928816-18-001652',
        '0000928816-18-001654', '0000051931-18-000823',
        '0000051931-18-000818', '0000894189-18-004917',
        '0000898432-18-000966', '0000898432-18-000968',
        '0000898432-18-000970', '0000898432-18-000971',
        '0000898432-18-000972', '0001144204-18-043917',
        '0001144204-18-046934', '0001144204-18-046936',
        '0001144204-18-046940', '0001144204-18-046943',
        '0001193125-18-263173', '0001193125-18-263198',
        '0001193125-18-263271', '0001615774-18-007808',
        '0000051931-18-000805', '0000051931-18-000811',
        '0000051931-18-000812', '0000928816-18-001652',
        '0000928816-18-001654', '0001387131-18-004331',
        '0000928816-19-001249', '0000928816-19-001253',
        '0000928816-19-001255', '0000928816-19-001271', '0000928816-19-001274']
<mask token>
f.processFormat3B('0000928816-19-001271', '0')

<mask token>


class Format3Pipeline:

    def processFormat3B(self, accession_no, documentId):
        try:
            content = Format3Parser.read_file(accession_no)
            clean_html = Format3html.cleanMe(content)
            clean_html = Format3Parser.segparsing(clean_html)
            tables = Format3Parser.returnTables(clean_html)
            print('type of tables elt is', type(tables))
            Format3Parser.df_toexcel(tables, accession_no, documentId,
                filePath=0)
        except Exception as e:
            print('Exception in parsing for accession no{} is {}: '.format(
                accession_no, e))


if __name__ == '__main__':
    accession_list = ['0001387131-18-004331', '0000928816-18-001652',
        '0000928816-18-001654', '0000051931-18-000823',
        '0000051931-18-000818', '0000894189-18-004917',
        '0000898432-18-000966', '0000898432-18-000968',
        '0000898432-18-000970', '0000898432-18-000971',
        '0000898432-18-000972', '0001144204-18-043917',
        '0001144204-18-046934', '0001144204-18-046936',
        '0001144204-18-046940', '0001144204-18-046943',
        '0001193125-18-263173', '0001193125-18-263198',
        '0001193125-18-263271', '0001615774-18-007808',
        '0000051931-18-000805', '0000051931-18-000811',
        '0000051931-18-000812', '0000928816-18-001652',
        '0000928816-18-001654', '0001387131-18-004331',
        '0000928816-19-001249', '0000928816-19-001253',
        '0000928816-19-001255', '0000928816-19-001271', '0000928816-19-001274']
new_acc_list = ['0001144204-19-037565', '0000928816-19-001314',
    '0000928816-19-001294', '0000928816-19-001286', '0000928816-19-001280',
    '0000928816-19-001274', '0000928816-19-001271', '0000928816-19-001268',
    '0000928816-19-001265', '0000928816-19-001261', '0000928816-19-001258',
    '0000928816-19-001255', '0000928816-19-001253', '0000928816-19-001249',
    '0000928816-19-001248', '0000928816-19-001245', '0000928816-19-001243',
    '0000898432-19-001139', '0000898432-19-001138', '0000898432-19-001137',
    '0000898432-19-001134', '0000898432-19-001133', '0000898432-19-001132',
    '0000898432-19-001130', '0001193125-19-229467', '0001387131-19-006459',
    '0001493152-19-010998', '0001615774-19-011383']
new_lot_12 = ['0000051931-18-000813', '0000051931-18-000814',
    '0000051931-18-000817', '0000051931-18-000819', '0000051931-18-000820',
    '0000051931-18-000821', '0000051931-18-000822', '0000051931-18-000824',
    '0000051931-18-000825', '0000051931-18-000829', '0000051931-18-000831',
    '0000051931-18-000832', '0000051931-18-000836', '0000051931-18-000837',
    '0000051931-18-000838', '0000051931-18-000840', '0000051931-18-000847',
    '0000894189-18-004917', '0000928816-18-001656', '0000928816-18-001658',
    '0000928816-18-001671', '0000928816-18-001672', '0000928816-18-001673',
    '0000928816-18-001674', '0000928816-18-001675', '0000928816-18-001676',
    '0000928816-18-001677', '0000928816-18-001678', '0000928816-18-001711',
    '0000928816-18-001714', '0000928816-18-001718', '0000928816-18-001720',
    '0000928816-18-001731', '0000928816-18-001734', '0000928816-18-001738',
    '0000928816-18-001739', '0001193125-18-252813', '0001615774-18-007808']
New = ['0000928816-19-001258', '0000928816-19-001261',
    '0000928816-19-001265', '0000928816-19-001268']
New_2312 = ['0000928816-18-001652', '0000928816-18-001654',
    '0001387131-18-004331', '0000928816-19-001268', '0000928816-19-001271',
    '0000928816-19-001274', '0000051931-18-000805', '0000051931-18-000811',
    '0000051931-18-000812', '0000051931-18-000813', '0000051931-18-000814',
    '0000051931-18-000817', '0000051931-18-000818', '0000051931-18-000819',
    '0000051931-18-000820', '0001387131-18-004331', '0000051931-18-000812',
    '0000051931-18-000818', '0000051931-18-000819', '0000928816-18-001652',
    '0000928816-18-001654', '0000928816-18-001656', '0000928816-18-001658',
    '0000051931-18-000822', '0001193125-18-252813', '0000051931-18-000823',
    '0000051931-18-000811', '0000928816-18-001674', '0000898432-18-000970',
    '0000898432-18-000971']
Newlot_26122019 = ['0000928816-19-001249', '0000928816-19-001253',
    '0000928816-19-001255', '0000928816-19-001258', '0000928816-19-001261',
    '0000928816-19-001265', '0000928816-19-001268', '0000928816-19-001271',
    '0000928816-19-001274', '0000051931-18-000805', '0000051931-18-000811',
    '0000051931-18-000812', '0000051931-18-000813', '0000051931-18-000814',
    '0000051931-18-000817', '0000051931-18-000818', '0000051931-18-000819',
    '0000051931-18-000820', '0000051931-18-000821', '0000051931-18-000822',
    '0000051931-18-000823', '0000051931-18-000824', '0000051931-18-000825',
    '0000051931-18-000829', '0000051931-18-000831', '0000051931-18-000832',
    '0000051931-18-000836', '0000051931-18-000837', '0000051931-18-000838',
    '0000051931-18-000840', '0000051931-18-000847', '0000894189-18-004917',
    '0000898432-18-000966', '0000898432-18-000968', '0000898432-18-000970',
    '0000898432-18-000971', '0000898432-18-000972', '0000928816-18-001652',
    '0000928816-18-001654', '0000928816-18-001656', '0000928816-18-001658',
    '0000928816-18-001671', '0000928816-18-001672', '0000928816-18-001673',
    '0000928816-18-001674', '0000928816-18-001675', '0000928816-18-001676',
    '0000928816-18-001677', '0000928816-18-001678', '0000928816-18-001711',
    '0000928816-18-001714', '0000928816-18-001718', '0000928816-18-001720',
    '0000928816-18-001731', '0000928816-18-001734', '0000928816-18-001738',
    '0000928816-18-001739', '0001144204-18-043917', '0001193125-18-252813',
    '0001387131-18-004331', '0001615774-18-007808', '0001144204-18-046934',
    '0001144204-18-046936', '0001144204-18-046940', '0001144204-18-046943',
    '0001193125-18-263173', '0001193125-18-263198', '0001193125-18-263271',
    '0000898432-19-001130', '0000898432-19-001133', '0000898432-19-001134',
    '0000898432-19-001137', '0000898432-19-001138', '0000928816-19-001243',
    '0000928816-19-001245', '0000928816-19-001248', '0000928816-19-001280',
    '0000928816-19-001286', '0000928816-19-001294', '0000928816-19-001314',
    '0001144204-19-037565', '0001193125-19-229467', '0001493152-19-010998']
ticker_missing = ['0001387131-18-004331', '0000051931-18-000812',
    '0000051931-18-000818', '0000051931-18-000819', '0000928816-18-001652',
    '0000928816-18-001654', '0000928816-18-001656', '0000928816-18-001658']
new_mismatch = ['0000051931-18-000822', '0001193125-18-252813',
    '0000051931-18-000811', '0000051931-18-000805', '0000928816-19-001268',
    '0000928816-19-001271', '0000928816-19-001274', '0000051931-18-000805',
    '0000051931-18-000811', '0000051931-18-000812', '0000051931-18-000813',
    '0000051931-18-000814', '0000051931-18-000817', '0000051931-18-000818',
    '0000051931-18-000819', '0000051931-18-000820', '0000928816-18-001652',
    '0000928816-18-001654', '0001387131-18-004331']
f = Format3Pipeline()
f.processFormat3B('0000928816-19-001271', '0')

import logging
from datetime import datetime
from Format3.format_3B.format3_html import Format3html
from config.config import CONFIG
from bs4 import BeautifulSoup
from Format3.format_3B.format3b_parser import Format3Parser


class Format3Pipeline:

    def processFormat3B(self, accession_no, documentId):
        try:
            content = Format3Parser.read_file(accession_no)
            clean_html = Format3html.cleanMe(content)
            clean_html = Format3Parser.segparsing(clean_html)
            tables = Format3Parser.returnTables(clean_html)
            print('type of tables elt is', type(tables))
            Format3Parser.df_toexcel(tables, accession_no, documentId,
                filePath=0)
        except Exception as e:
            print('Exception in parsing for accession no{} is {}: '.format(
                accession_no, e))


if __name__ == '__main__':
    accession_list = ['0001387131-18-004331', '0000928816-18-001652',
        '0000928816-18-001654', '0000051931-18-000823',
        '0000051931-18-000818', '0000894189-18-004917',
        '0000898432-18-000966', '0000898432-18-000968',
        '0000898432-18-000970', '0000898432-18-000971',
        '0000898432-18-000972', '0001144204-18-043917',
        '0001144204-18-046934', '0001144204-18-046936',
        '0001144204-18-046940', '0001144204-18-046943',
        '0001193125-18-263173', '0001193125-18-263198',
        '0001193125-18-263271', '0001615774-18-007808',
        '0000051931-18-000805', '0000051931-18-000811',
        '0000051931-18-000812', '0000928816-18-001652',
        '0000928816-18-001654', '0001387131-18-004331',
        '0000928816-19-001249', '0000928816-19-001253',
        '0000928816-19-001255', '0000928816-19-001271', '0000928816-19-001274']
new_acc_list = ['0001144204-19-037565', '0000928816-19-001314',
    '0000928816-19-001294', '0000928816-19-001286', '0000928816-19-001280',
    '0000928816-19-001274', '0000928816-19-001271', '0000928816-19-001268',
    '0000928816-19-001265', '0000928816-19-001261', '0000928816-19-001258',
    '0000928816-19-001255', '0000928816-19-001253', '0000928816-19-001249',
    '0000928816-19-001248', '0000928816-19-001245', '0000928816-19-001243',
    '0000898432-19-001139', '0000898432-19-001138', '0000898432-19-001137',
    '0000898432-19-001134', '0000898432-19-001133', '0000898432-19-001132',
    '0000898432-19-001130', '0001193125-19-229467', '0001387131-19-006459',
    '0001493152-19-010998', '0001615774-19-011383']
new_lot_12 = ['0000051931-18-000813', '0000051931-18-000814',
    '0000051931-18-000817', '0000051931-18-000819', '0000051931-18-000820',
    '0000051931-18-000821', '0000051931-18-000822', '0000051931-18-000824',
    '0000051931-18-000825', '0000051931-18-000829', '0000051931-18-000831',
    '0000051931-18-000832', '0000051931-18-000836', '0000051931-18-000837',
    '0000051931-18-000838', '0000051931-18-000840', '0000051931-18-000847',
    '0000894189-18-004917', '0000928816-18-001656', '0000928816-18-001658',
    '0000928816-18-001671', '0000928816-18-001672', '0000928816-18-001673',
    '0000928816-18-001674', '0000928816-18-001675', '0000928816-18-001676',
    '0000928816-18-001677', '0000928816-18-001678', '0000928816-18-001711',
    '0000928816-18-001714', '0000928816-18-001718', '0000928816-18-001720',
    '0000928816-18-001731', '0000928816-18-001734', '0000928816-18-001738',
    '0000928816-18-001739', '0001193125-18-252813', '0001615774-18-007808']
New = ['0000928816-19-001258', '0000928816-19-001261',
    '0000928816-19-001265', '0000928816-19-001268']
New_2312 = ['0000928816-18-001652', '0000928816-18-001654',
    '0001387131-18-004331', '0000928816-19-001268', '0000928816-19-001271',
    '0000928816-19-001274', '0000051931-18-000805', '0000051931-18-000811',
    '0000051931-18-000812', '0000051931-18-000813', '0000051931-18-000814',
    '0000051931-18-000817', '0000051931-18-000818', '0000051931-18-000819',
    '0000051931-18-000820', '0001387131-18-004331', '0000051931-18-000812',
    '0000051931-18-000818', '0000051931-18-000819', '0000928816-18-001652',
    '0000928816-18-001654', '0000928816-18-001656', '0000928816-18-001658',
    '0000051931-18-000822', '0001193125-18-252813', '0000051931-18-000823',
    '0000051931-18-000811', '0000928816-18-001674', '0000898432-18-000970',
    '0000898432-18-000971']
Newlot_26122019 = ['0000928816-19-001249', '0000928816-19-001253',
    '0000928816-19-001255', '0000928816-19-001258', '0000928816-19-001261',
    '0000928816-19-001265', '0000928816-19-001268', '0000928816-19-001271',
    '0000928816-19-001274', '0000051931-18-000805', '0000051931-18-000811',
    '0000051931-18-000812', '0000051931-18-000813', '0000051931-18-000814',
    '0000051931-18-000817', '0000051931-18-000818', '0000051931-18-000819',
    '0000051931-18-000820', '0000051931-18-000821', '0000051931-18-000822',
    '0000051931-18-000823', '0000051931-18-000824', '0000051931-18-000825',
    '0000051931-18-000829', '0000051931-18-000831', '0000051931-18-000832',
    '0000051931-18-000836', '0000051931-18-000837', '0000051931-18-000838',
    '0000051931-18-000840', '0000051931-18-000847', '0000894189-18-004917',
    '0000898432-18-000966', '0000898432-18-000968', '0000898432-18-000970',
    '0000898432-18-000971', '0000898432-18-000972', '0000928816-18-001652',
    '0000928816-18-001654', '0000928816-18-001656', '0000928816-18-001658',
    '0000928816-18-001671', '0000928816-18-001672', '0000928816-18-001673',
    '0000928816-18-001674', '0000928816-18-001675', '0000928816-18-001676',
    '0000928816-18-001677', '0000928816-18-001678', '0000928816-18-001711',
    '0000928816-18-001714', '0000928816-18-001718', '0000928816-18-001720',
    '0000928816-18-001731', '0000928816-18-001734', '0000928816-18-001738',
    '0000928816-18-001739', '0001144204-18-043917', '0001193125-18-252813',
    '0001387131-18-004331', '0001615774-18-007808', '0001144204-18-046934',
    '0001144204-18-046936', '0001144204-18-046940', '0001144204-18-046943',
    '0001193125-18-263173', '0001193125-18-263198', '0001193125-18-263271',
    '0000898432-19-001130', '0000898432-19-001133', '0000898432-19-001134',
    '0000898432-19-001137', '0000898432-19-001138', '0000928816-19-001243',
    '0000928816-19-001245', '0000928816-19-001248', '0000928816-19-001280',
    '0000928816-19-001286', '0000928816-19-001294', '0000928816-19-001314',
    '0001144204-19-037565', '0001193125-19-229467', '0001493152-19-010998']
ticker_missing = ['0001387131-18-004331', '0000051931-18-000812',
    '0000051931-18-000818', '0000051931-18-000819', '0000928816-18-001652',
    '0000928816-18-001654', '0000928816-18-001656', '0000928816-18-001658']
new_mismatch = ['0000051931-18-000822', '0001193125-18-252813',
    '0000051931-18-000811', '0000051931-18-000805', '0000928816-19-001268',
    '0000928816-19-001271', '0000928816-19-001274', '0000051931-18-000805',
    '0000051931-18-000811', '0000051931-18-000812', '0000051931-18-000813',
    '0000051931-18-000814', '0000051931-18-000817', '0000051931-18-000818',
    '0000051931-18-000819', '0000051931-18-000820', '0000928816-18-001652',
    '0000928816-18-001654', '0001387131-18-004331']
f = Format3Pipeline()
f.processFormat3B('0000928816-19-001271', '0')

import logging
from datetime import datetime

from Format3.format_3B.format3_html import Format3html
from config.config import CONFIG
from bs4 import BeautifulSoup

from Format3.format_3B.format3b_parser import Format3Parser


class Format3Pipeline:

    # def __init__(self):
        # logging.config.fileConfig('loggerconfig.conf')
        # self.logger = logging.getLogger('MainLogger')
        # fh = logging.FileHandler(CONFIG["Path"]["log_path"]+'{:%Y-%m-%d}.log'.format(datetime.now()))
        # formatter = logging.Formatter('%(asctime)s | %(levelname)-8s | %(lineno)04d | %(message)s')
        # fh.setFormatter(formatter)
        # self.logger.addHandler(fh)

    def processFormat3B(self,accession_no,documentId):
        try:
            content=Format3Parser.read_file(accession_no)
            clean_html=Format3html.cleanMe(content)
            clean_html=Format3Parser.segparsing(clean_html)
            # clean_html=clean_html.replace("<div>","")
            # clean_html=clean_html.replace("</div>","")
            # print(clean_html,"\n\n\n")
            tables=Format3Parser.returnTables(clean_html)
            print("type of tables elt is",type(tables))
            Format3Parser.df_toexcel(tables,accession_no,documentId,filePath=0)
            #Format3Parser.df_tosql(df,accession_no,documentId)
        except Exception as e:
            print("Exception in parsing for accession no{} is {}: ".format(accession_no,e))

if __name__=="__main__":

    accession_list=["0001387131-18-004331",
    "0000928816-18-001652",
    "0000928816-18-001654","0000051931-18-000823",
    "0000051931-18-000818","0000894189-18-004917",
"0000898432-18-000966",
"0000898432-18-000968",
"0000898432-18-000970",
"0000898432-18-000971",
"0000898432-18-000972",
"0001144204-18-043917",
"0001144204-18-046934",
"0001144204-18-046936",
"0001144204-18-046940",
"0001144204-18-046943",
"0001193125-18-263173",
"0001193125-18-263198",
"0001193125-18-263271",
"0001615774-18-007808","0000051931-18-000805",
"0000051931-18-000811",
"0000051931-18-000812","0000928816-18-001652",
"0000928816-18-001654",
"0001387131-18-004331","0000928816-19-001249",
"0000928816-19-001253",
"0000928816-19-001255","0000928816-19-001271",
"0000928816-19-001274"]

new_acc_list=["0001144204-19-037565",
"0000928816-19-001314",
"0000928816-19-001294",
"0000928816-19-001286",
"0000928816-19-001280",
"0000928816-19-001274",
"0000928816-19-001271",
"0000928816-19-001268",
"0000928816-19-001265",
"0000928816-19-001261",
"0000928816-19-001258",
"0000928816-19-001255",
"0000928816-19-001253",
"0000928816-19-001249",
"0000928816-19-001248",
"0000928816-19-001245",
"0000928816-19-001243",
"0000898432-19-001139",
"0000898432-19-001138",
"0000898432-19-001137",
"0000898432-19-001134",
"0000898432-19-001133",
"0000898432-19-001132",
"0000898432-19-001130",
"0001193125-19-229467",
"0001387131-19-006459",
"0001493152-19-010998",
"0001615774-19-011383"
]

new_lot_12=["0000051931-18-000813",
"0000051931-18-000814",
"0000051931-18-000817",
"0000051931-18-000819",
"0000051931-18-000820",
"0000051931-18-000821",
"0000051931-18-000822",
"0000051931-18-000824",
"0000051931-18-000825",
"0000051931-18-000829",
"0000051931-18-000831",
"0000051931-18-000832",
"0000051931-18-000836",
"0000051931-18-000837",
"0000051931-18-000838",
"0000051931-18-000840",
"0000051931-18-000847",
"0000894189-18-004917",
"0000928816-18-001656",
"0000928816-18-001658",
"0000928816-18-001671",
"0000928816-18-001672",
"0000928816-18-001673",
"0000928816-18-001674",
"0000928816-18-001675",
"0000928816-18-001676",
"0000928816-18-001677",
"0000928816-18-001678",
"0000928816-18-001711",
"0000928816-18-001714",
"0000928816-18-001718",
"0000928816-18-001720",
"0000928816-18-001731",
"0000928816-18-001734",
"0000928816-18-001738",
"0000928816-18-001739",
"0001193125-18-252813",
"0001615774-18-007808"
]

New=["0000928816-19-001258",
"0000928816-19-001261",
"0000928816-19-001265",
"0000928816-19-001268"]

New_2312=[
"0000928816-18-001652",
"0000928816-18-001654",
"0001387131-18-004331",
"0000928816-19-001268",
"0000928816-19-001271",
"0000928816-19-001274",
"0000051931-18-000805",
"0000051931-18-000811",
"0000051931-18-000812",
"0000051931-18-000813",
"0000051931-18-000814",
"0000051931-18-000817",
"0000051931-18-000818",
"0000051931-18-000819",
"0000051931-18-000820",
"0001387131-18-004331",
"0000051931-18-000812",
"0000051931-18-000818",
"0000051931-18-000819",
"0000928816-18-001652",
"0000928816-18-001654",
"0000928816-18-001656",
"0000928816-18-001658",
"0000051931-18-000822",
"0001193125-18-252813",
"0000051931-18-000823",
"0000051931-18-000811",
"0000928816-18-001674",
"0000898432-18-000970",
"0000898432-18-000971"
]

Newlot_26122019=["0000928816-19-001249",
"0000928816-19-001253",
"0000928816-19-001255",
"0000928816-19-001258",
"0000928816-19-001261",
"0000928816-19-001265",
"0000928816-19-001268",
"0000928816-19-001271",
"0000928816-19-001274",
"0000051931-18-000805",
"0000051931-18-000811",
"0000051931-18-000812",
"0000051931-18-000813",
"0000051931-18-000814",
"0000051931-18-000817",
"0000051931-18-000818",
"0000051931-18-000819",
"0000051931-18-000820",
"0000051931-18-000821",
"0000051931-18-000822",
"0000051931-18-000823",
"0000051931-18-000824",
"0000051931-18-000825",
"0000051931-18-000829",
"0000051931-18-000831",
"0000051931-18-000832",
"0000051931-18-000836",
"0000051931-18-000837",
"0000051931-18-000838",
"0000051931-18-000840",
"0000051931-18-000847",
"0000894189-18-004917",
"0000898432-18-000966",
"0000898432-18-000968",
"0000898432-18-000970",
"0000898432-18-000971",
"0000898432-18-000972",
"0000928816-18-001652",
"0000928816-18-001654",
"0000928816-18-001656",
"0000928816-18-001658",
"0000928816-18-001671",
"0000928816-18-001672",
"0000928816-18-001673",
"0000928816-18-001674",
"0000928816-18-001675",
"0000928816-18-001676",
"0000928816-18-001677",
"0000928816-18-001678",
"0000928816-18-001711",
"0000928816-18-001714",
"0000928816-18-001718",
"0000928816-18-001720",
"0000928816-18-001731",
"0000928816-18-001734",
"0000928816-18-001738",
"0000928816-18-001739",
"0001144204-18-043917",
"0001193125-18-252813",
"0001387131-18-004331",
"0001615774-18-007808",
"0001144204-18-046934",
"0001144204-18-046936",
"0001144204-18-046940",
"0001144204-18-046943",
"0001193125-18-263173",
"0001193125-18-263198",
"0001193125-18-263271",
"0000898432-19-001130",
"0000898432-19-001133",
"0000898432-19-001134",
"0000898432-19-001137",
"0000898432-19-001138",
"0000928816-19-001243",
"0000928816-19-001245",
"0000928816-19-001248",
"0000928816-19-001280",
"0000928816-19-001286",
"0000928816-19-001294",
"0000928816-19-001314",
"0001144204-19-037565",
"0001193125-19-229467",
"0001493152-19-010998"]


ticker_missing=['0001387131-18-004331',
'0000051931-18-000812',
'0000051931-18-000818',
'0000051931-18-000819',
'0000928816-18-001652',
'0000928816-18-001654',
'0000928816-18-001656',
'0000928816-18-001658'
]

new_mismatch=['0000051931-18-000822',
              '0001193125-18-252813',
              '0000051931-18-000811',
              '0000051931-18-000805',
              '0000928816-19-001268',
              '0000928816-19-001271',
              '0000928816-19-001274',
              '0000051931-18-000805',
              '0000051931-18-000811',
              '0000051931-18-000812',
              '0000051931-18-000813',
              '0000051931-18-000814',
              '0000051931-18-000817',
              '0000051931-18-000818',
              '0000051931-18-000819',
              '0000051931-18-000820',
              '0000928816-18-001652',
              '0000928816-18-001654',
              '0001387131-18-004331',
              ]
f = Format3Pipeline()

# for accession_no in new_mismatch:
#     try:
#         f = Format3Pipeline()
#         f.processFormat3B(accession_no,'0')
#     except:
#         print("error for", accession_no)

f.processFormat3B("0000928816-19-001271",'0')
# f.processFormat3B("0000051931-18-000805",'0')
# # # f.processFormat3B("0000928816-18-001654",'0')
# f.processFormat3B("0001387131-18-004331",'0')