from HTMLParser import HTMLParser
from pprint import pprint

"""
File for testing extraction of JSON from html format
potential to be default case for structured data parsing

source:
http://www.xavierdupre.fr/blog/2013-10-27_nojs.html
"""
class markup_parser(HTMLParser):
    def __init__(self, raise_exception=False) :
        HTMLParser.__init__(self)
        self.json  = { }
        self.path = []
        self.cur  = self.json
        self.line = 0
        self.raise_exception = raise_exception

    def to_json(self, content):
        self.feed(content)
        return self.json

    def handle_starttag(self, tag, attrs):
        self.path.append(tag)
        attrs = { k:v for k,v in attrs }
        if tag in self.cur :
            if isinstance(self.cur[tag],list) :
                self.cur[tag].append(  { "__parent__": self.cur } )
                self.cur = self.cur[tag][-1]
            else :
                self.cur[tag] = [self.cur[tag]]
                self.cur[tag].append(  { "__parent__": self.cur } )
                self.cur = self.cur[tag][-1]
        else :
            self.cur[tag] = { "__parent__": self.cur }
            self.cur = self.cur[tag]

        for a,v in attrs.items():
            self.cur["#" + a] = v
        self.cur[""] = ""

    def handle_endtag(self, tag):
        if tag != self.path[-1] and self.raise_exception :
            raise Exception("html is malformed around line: {0} (it might be because of a tag <br>, <hr>, <img .. > not closed)".format(self.line))
        del self.path[-1]
        memo = self.cur
        self.cur = self.cur["__parent__"]
        self.clean(memo)

    def handle_data(self, data):
        self.line += data.count("\n")
        if "" in self.cur :
            self.cur[""] += data

    def clean(self, values):
        keys = list(values.keys())
        for k in keys:
            v = values[k]
            if isinstance(v, str) :
                #print ("clean", k,[v])
                c = v.strip(" \n\r\t")
                if c != v :
                    if len(c) > 0 :
                        values[k] = c
                    else :
                        del values[k]
        del values["__parent__"]

if __name__ == '__main__':
    """
    debugging demo
    """
    with open("outfile.html") as file:
        content = file.read()
        mkup = markup_parser()
        js = mkup.to_json(content)
        pprint(js)
3bdd4c36a36da032d33d6d9b60a2d9f325863ca7
<mask token>


class markup_parser(HTMLParser):
    <mask token>

    def to_json(self, content):
        self.feed(content)
        return self.json

    def handle_starttag(self, tag, attrs):
        self.path.append(tag)
        attrs = {k: v for k, v in attrs}
        if tag in self.cur:
            if isinstance(self.cur[tag], list):
                self.cur[tag].append({'__parent__': self.cur})
                self.cur = self.cur[tag][-1]
            else:
                self.cur[tag] = [self.cur[tag]]
                self.cur[tag].append({'__parent__': self.cur})
                self.cur = self.cur[tag][-1]
        else:
            self.cur[tag] = {'__parent__': self.cur}
            self.cur = self.cur[tag]
        for a, v in attrs.items():
            self.cur['#' + a] = v
        self.cur[''] = ''

    def handle_endtag(self, tag):
        if tag != self.path[-1] and self.raise_exception:
            raise Exception(
                'html is malformed around line: {0} (it might be because of a tag <br>, <hr>, <img .. > not closed)'
                .format(self.line))
        del self.path[-1]
        memo = self.cur
        self.cur = self.cur['__parent__']
        self.clean(memo)

    def handle_data(self, data):
        self.line += data.count('\n')
        if '' in self.cur:
            self.cur[''] += data
    <mask token>


<mask token>

<mask token>


class markup_parser(HTMLParser):

    def __init__(self, raise_exception=False):
        HTMLParser.__init__(self)
        self.json = {}
        self.path = []
        self.cur = self.json
        self.line = 0
        self.raise_exception = raise_exception

    def to_json(self, content):
        self.feed(content)
        return self.json

    def handle_starttag(self, tag, attrs):
        self.path.append(tag)
        attrs = {k: v for k, v in attrs}
        if tag in self.cur:
            if isinstance(self.cur[tag], list):
                self.cur[tag].append({'__parent__': self.cur})
                self.cur = self.cur[tag][-1]
            else:
                self.cur[tag] = [self.cur[tag]]
                self.cur[tag].append({'__parent__': self.cur})
                self.cur = self.cur[tag][-1]
        else:
            self.cur[tag] = {'__parent__': self.cur}
            self.cur = self.cur[tag]
        for a, v in attrs.items():
            self.cur['#' + a] = v
        self.cur[''] = ''

    def handle_endtag(self, tag):
        if tag != self.path[-1] and self.raise_exception:
            raise Exception(
                'html is malformed around line: {0} (it might be because of a tag <br>, <hr>, <img .. > not closed)'
                .format(self.line))
        del self.path[-1]
        memo = self.cur
        self.cur = self.cur['__parent__']
        self.clean(memo)

    def handle_data(self, data):
        self.line += data.count('\n')
        if '' in self.cur:
            self.cur[''] += data
    <mask token>


<mask token>

<mask token>


class markup_parser(HTMLParser):

    def __init__(self, raise_exception=False):
        HTMLParser.__init__(self)
        self.json = {}
        self.path = []
        self.cur = self.json
        self.line = 0
        self.raise_exception = raise_exception

    def to_json(self, content):
        self.feed(content)
        return self.json

    def handle_starttag(self, tag, attrs):
        self.path.append(tag)
        attrs = {k: v for k, v in attrs}
        if tag in self.cur:
            if isinstance(self.cur[tag], list):
                self.cur[tag].append({'__parent__': self.cur})
                self.cur = self.cur[tag][-1]
            else:
                self.cur[tag] = [self.cur[tag]]
                self.cur[tag].append({'__parent__': self.cur})
                self.cur = self.cur[tag][-1]
        else:
            self.cur[tag] = {'__parent__': self.cur}
            self.cur = self.cur[tag]
        for a, v in attrs.items():
            self.cur['#' + a] = v
        self.cur[''] = ''

    def handle_endtag(self, tag):
        if tag != self.path[-1] and self.raise_exception:
            raise Exception(
                'html is malformed around line: {0} (it might be because of a tag <br>, <hr>, <img .. > not closed)'
                .format(self.line))
        del self.path[-1]
        memo = self.cur
        self.cur = self.cur['__parent__']
        self.clean(memo)

    def handle_data(self, data):
        self.line += data.count('\n')
        if '' in self.cur:
            self.cur[''] += data

    def clean(self, values):
        keys = list(values.keys())
        for k in keys:
            v = values[k]
            if isinstance(v, str):
                c = v.strip(' \n\r\t')
                if c != v:
                    if len(c) > 0:
                        values[k] = c
                    else:
                        del values[k]
        del values['__parent__']


if __name__ == '__main__':
    """
    debugging demo
    """
    with open('outfile.html') as file:
        content = file.read()
        mkup = markup_parser()
        js = mkup.to_json(content)
        pprint(js)

from HTMLParser import HTMLParser
from pprint import pprint
<mask token>


class markup_parser(HTMLParser):

    def __init__(self, raise_exception=False):
        HTMLParser.__init__(self)
        self.json = {}
        self.path = []
        self.cur = self.json
        self.line = 0
        self.raise_exception = raise_exception

    def to_json(self, content):
        self.feed(content)
        return self.json

    def handle_starttag(self, tag, attrs):
        self.path.append(tag)
        attrs = {k: v for k, v in attrs}
        if tag in self.cur:
            if isinstance(self.cur[tag], list):
                self.cur[tag].append({'__parent__': self.cur})
                self.cur = self.cur[tag][-1]
            else:
                self.cur[tag] = [self.cur[tag]]
                self.cur[tag].append({'__parent__': self.cur})
                self.cur = self.cur[tag][-1]
        else:
            self.cur[tag] = {'__parent__': self.cur}
            self.cur = self.cur[tag]
        for a, v in attrs.items():
            self.cur['#' + a] = v
        self.cur[''] = ''

    def handle_endtag(self, tag):
        if tag != self.path[-1] and self.raise_exception:
            raise Exception(
                'html is malformed around line: {0} (it might be because of a tag <br>, <hr>, <img .. > not closed)'
                .format(self.line))
        del self.path[-1]
        memo = self.cur
        self.cur = self.cur['__parent__']
        self.clean(memo)

    def handle_data(self, data):
        self.line += data.count('\n')
        if '' in self.cur:
            self.cur[''] += data

    def clean(self, values):
        keys = list(values.keys())
        for k in keys:
            v = values[k]
            if isinstance(v, str):
                c = v.strip(' \n\r\t')
                if c != v:
                    if len(c) > 0:
                        values[k] = c
                    else:
                        del values[k]
        del values['__parent__']


if __name__ == '__main__':
    """
    debugging demo
    """
    with open('outfile.html') as file:
        content = file.read()
        mkup = markup_parser()
        js = mkup.to_json(content)
        pprint(js)

from HTMLParser import HTMLParser
from pprint import pprint

"""
File for testing extraction of JSON from html format
potential to be default case for structured data parsing

source:
http://www.xavierdupre.fr/blog/2013-10-27_nojs.html
"""
class markup_parser(HTMLParser):
    def __init__(self, raise_exception=False) :
        HTMLParser.__init__(self)
        self.json  = { }
        self.path = []
        self.cur  = self.json
        self.line = 0
        self.raise_exception = raise_exception

    def to_json(self, content):
        self.feed(content)
        return self.json

    def handle_starttag(self, tag, attrs):
        self.path.append(tag)
        attrs = { k:v for k,v in attrs }
        if tag in self.cur :
            if isinstance(self.cur[tag],list) :
                self.cur[tag].append(  { "__parent__": self.cur } )
                self.cur = self.cur[tag][-1]
            else :
                self.cur[tag] = [self.cur[tag]]
                self.cur[tag].append(  { "__parent__": self.cur } )
                self.cur = self.cur[tag][-1]
        else :
            self.cur[tag] = { "__parent__": self.cur }
            self.cur = self.cur[tag]

        for a,v in attrs.items():
            self.cur["#" + a] = v
        self.cur[""] = ""

    def handle_endtag(self, tag):
        if tag != self.path[-1] and self.raise_exception :
            raise Exception("html is malformed around line: {0} (it might be because of a tag <br>, <hr>, <img .. > not closed)".format(self.line))
        del self.path[-1]
        memo = self.cur
        self.cur = self.cur["__parent__"]
        self.clean(memo)

    def handle_data(self, data):
        self.line += data.count("\n")
        if "" in self.cur :
            self.cur[""] += data

    def clean(self, values):
        keys = list(values.keys())
        for k in keys:
            v = values[k]
            if isinstance(v, str) :
                #print ("clean", k,[v])
                c = v.strip(" \n\r\t")
                if c != v :
                    if len(c) > 0 :
                        values[k] = c
                    else :
                        del values[k]
        del values["__parent__"]

if __name__ == '__main__':
    """
    debugging demo
    """
    with open("outfile.html") as file:
        content = file.read()
        mkup = markup_parser()
        js = mkup.to_json(content)
        pprint(js)