#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""dp2ppgen

Usage:
  dp2ppgen [options] <infile> [<outfile>]
  dp2ppgen -h | --help
  dp2ppgen --version

Translates pgdp.org formatted text files into ppgen syntax.

Examples:
  dp2ppgen book.txt
  dp2ppgen book.txt book-src.txt

Options:
  --boilerplate                Pastes contents of header.txt and footer.txt to start and end
  -c, --chapters               Convert chapter headings into ppgen style chapter headings
  --config=<config>            Use the set of options in the given configuration file
  --chaptermaxlines=<max>      Max lines a chapter can be, anything larger is not a chapter [default: 15]
  -d, --dryrun                 Run through conversions but do not write out result
  -e, --sections               Convert section headings into ppgen style section headings
  --sectionmaxlines=<max>      Max lines a section can be, ianything larger is not a section [default: 3]
  -f, --footnotes              Convert footnotes into ppgen format
  --fnautonum                  Use ppgen autonumbering for generated anchors and .fn statements
  --fndest=<fndest>            Where to relocate footnotes (paragraphend, chapterend, bookend)
  --lzdestt=<lzdestt>          Where to place footnote landing zones for text output (chapterend, bookend)
  --lzdesth=<lzdesth>          Where to place footnote landing zones for HTML output (chapterend, bookend)
  --fixup                      Perform guiguts style fixup operations
  --force                      Ignore markup errors and force operation
  -i, --illustrations          Convert raw [Illustration] tags into ppgen .il/.ca markup
  -j, --joinspanned            Join hypenations (-* *-) and formatting markup (/* */ /# #/) that spans page breaks
  --autofixhyphens             Analyze hyphenated word usage and replace joined hyphenations with best fit (if one exists)
  -k, --keeporiginal           On any conversion keep original text as a comment
  -p, --pages                  Convert page breaks into ppgen // 001.png style, add .pn statements and comment out [Blank Page] lines
  -q, --quiet                  Print less text
  -r, --report=[txt,html,csv]  Perform various analysis on the input file
  -s, --sidenotes              Convert sidenotes into ppgen format
  --snkeepbreaks               Keep exact line endings for multi-line sidenotes
  --detectmarkup               Best guess what out of line markup /* */ /# #/ represent (table, toc, poetry, etc..)
  --tnote                      Generate transcribers note
  -m, --markup                 Convert out of line markup /* */ /# #/ into ppgen format
  -v, --verbose                Print more text
  -h, --help                   Show help
  --utf8                       Convert characters to UTF8
  --version                    Show version
"""

from docopt import docopt
import glob
import re
import os
import sys
import logging
import tempfile
import subprocess
import shlex
import json
from PIL import Image


__appname__ = "dp2ppgen"
__author__  = "David Maranhao"
__license__ = "MIT"
__version__ = "0.2.0" # MAJOR.MINOR.PATCH | http://semver.org


markupTypes = {
    'nf': {
        'alias': None,
        'count': None,
    },
    'table': {
        'alias': ('t', '$'),
        'count': None,
    },
    'ta': {
        'alias': None,
        'count': None,
    },
    'toc': {
        'alias': ('x', 'X'),
        'count': None,
    },
    'title': {
        'alias': ('titlepage', 'f', 'F', 'frontmatter'),
        'count': None,
    },
    'poetry': {
        'alias': ('poem', 'p', 'P'),
        'count': None,
    },
    'index': {
        'alias': ('idx'),
        'count': None,
    },
    'blockquote': {
        'alias': ('bq'),
        'count': None,
    },
    'hangingindent': {
        'alias': ('hang', 'hi'),
        'count': None,
    },
    'halftitle': {
        'alias': ('ht'),
        'count': None,
    },
    'signature': {
        'alias': ('sig'),
        'count': None,
    },
    #TODO
    # L - Unsigned List
}


# Limited check for syntax errors in dp markup of input file
def validateDpMarkup(inBuf):

    # TODO, someone must have written a more thorough version of this already.. use that instead

    logging.info("Checking input file for markup errors")

    inBuf = removeTrailingSpaces(inBuf)

    formattingStack = []
    lineNum = 0
    errorCount = 0
    while lineNum < len(inBuf):

        # Detect unbalanced out-of-line formatting markup /# #/ /* */
        m = re.match(r"\/(\*|\#)", inBuf[lineNum])
        if m:
            d = ({'ln':lineNum+1, 'v':"/{}".format(m.group(1))})
            formattingStack.append(d)

        m = re.match(r"(\*|\#)\/", inBuf[lineNum])
        if m:
            v = m.group(1)
            if len(formattingStack) == 0 or formattingStack[-1]['v'] != "/{}".format(v):
                errorCount += 1
                if len(formattingStack) == 0:
                    logging.error("Line {}: Unexpected {}/".format(lineNum+1, v))
                else:
                    logging.error("Line {}: Unexpected {}/, previous ({}:{})".format(lineNum+1, v, formattingStack[-1]['ln'], formattingStack[-1]['v']))
            else:
                formattingStack.pop()


#       # Check balance of [], {}, (), <i></i>
#       m = re.findall(r"(\[|\]|\{|\}|\(|\)|<\/?\w+>)", inBuf[lineNum])

#       # Check balance of [], {}, <i></i>
#       m = re.findall(r"(\[|\]|\{|\}|<\/?\w+>)", inBuf[lineNum])

        # Check balance of [], <i></i>
        m = re.findall(r"(\[|\]|<\/?[^>]>)", inBuf[lineNum])
        for v in m:

            if v == "<tb>": # ignore
                pass

            elif v == "]": # closing markup
                if len(formattingStack) == 0 or formattingStack[-1]['v'] != "[":
                    errorCount += 1
                    if len(formattingStack) == 0:
                        logging.error("Line {}: Unexpected {}".format(lineNum+1, v))
                    else:
                        logging.error("Line {}: Unexpected {}, previous ({}:{})".format(lineNum+1, v, formattingStack[-1]['ln'], formattingStack[-1]['v']))
                else:
                    formattingStack.pop()

#           elif v == "}": # closing markup
#               if len(formattingStack) == 0 or formattingStack[-1]['v'] != "{":
#                   errorCount += 1
#                   if len(formattingStack) == 0:
#                       logging.error("Line {}: Unexpected {}".format(lineNum+1, v))
#                   else:
#                       logging.error("Line {}: Unexpected {}, previous ({}:{})".format(lineNum+1, v, formattingStack[-1]['ln'], formattingStack[-1]['v']))
#                       logging.debug("{}".format(formattingStack))
#               else:
#                   formattingStack.pop()

            # Disabled as this will get false positives from diacratic markup [)x] and won't affect conversion anyways
#               if len(formattingStack) == 0 or formattingStack[-1]['v'] != "(":
#                   errorCount += 1
#                   if len(formattingStack) == 0:
#                       logging.error("Line {}: Unexpected {}".format(lineNum+1, v))
#                   else:
#                       logging.error("Line {}: Unexpected {}, previous ({}:{})".format(lineNum+1, v, formattingStack[-1]['ln'], formattingStack[-1]['v']))
#                       logging.debug("{}".format(formattingStack))
#               else:
#                   formattingStack.pop()

            elif "/" in v: # closing markup
                v2 = re.sub("/", "", v)
                if len(formattingStack) == 0 or formattingStack[-1]['v'] != v2:
                    errorCount += 1
                    if len(formattingStack) == 0:
                        logging.error("Line {}: Unexpected {}".format(lineNum+1, v))
                    else:
                        logging.error("Line {}: Unexpected {}, previous ({}:{})".format(lineNum+1, v, formattingStack[-1]['ln'], formattingStack[-1]['v']))
                else:
                    formattingStack.pop()

            else:
                d = ({'ln':lineNum+1, 'v':v})
                formattingStack.append(d)


        # Check for specific issues that have caused conversion issues in the past

        # Single line [Footnote] does not end at closing ]
        # ex. [Footnote 1: Duine, <i>Saints de Domnonée</i>, pp. 5-12].
        if re.match(r"\*?\[Footnote(.*)\]\*?.*$", inBuf[lineNum]):
            if inBuf[lineNum].count('[') - inBuf[lineNum].count(']') == 0: # ignore multiline footnotes with proofer notes or some other [] markup within them
                if not (inBuf[lineNum][-1] == ']' or inBuf[lineNum][-2:] == ']*'):
                    errorCount += 1
                    logging.error("Line {}: Extra characters found after closing ']' in [Footnote]\n       {}".format(lineNum+1, inBuf[lineNum]))

        # Extra text after out-of-line formatting markup
        # ex. /*[**new stanza?]
#       m = re.match(r"(\/\*|\/\#|\*\/|\#\/)(.+)", inBuf[lineNum])
#       if m and m.group(2) not in markupTypes['/*'] and m.group(2) not in markupTypes['/#']:
#           errorCount += 1
#           logging.error("Line {}: Extra text after out-of-line formatting markup\n       {}".format(lineNum+1, inBuf[lineNum]))

        lineNum += 1

        # Chapters
        # Sections

    # Look for unresolved <i></i>, [], {}
    if formattingStack:
        errorCount += 1
        logging.error("Reached end of file with unresolved formatting markup, (probably due to previous markup error(s))")

        if errorCount == 1:
            logging.error("Unresolved markup:")
            s = "Line {}: '{}'".format(formattingStack[0]['ln'], formattingStack[0]['v'])
            for v in formattingStack[1:]:
                s += ", Line {}: '{}'".format(v['ln'], v['v'])
            logging.error(s)
        else:
            logging.debug(formattingStack)

    if errorCount > 0:
        logging.info("Found {} markup errors".format(errorCount))

    return errorCount


# Format helper function, truncate to width and indicate truncation occured with ...
def truncate(string, width):
    if len(string) > width:
        string = string[:width-3] + '...'
    return string


# Removes trailing spaces and tabs from an array of strings
def removeTrailingSpaces(inBuf):
    outBuf = []

    for line in inBuf:
        outBuf.append(line.rstrip(" \t"))

    return outBuf


# Replace : [Blank Page]
# with    : // [Blank Page]
def processBlankPages(inBuf, keepOriginal):
    outBuf = []
    lineNum = 0
    count = 0

    logging.info("Processing blank pages")

    while lineNum < len(inBuf):
        if inBuf[lineNum].startswith("[Blank Page]"):
            if keepOriginal:
                outBuf.append("// *** DP2PPGEN ORIGINAL: {}".format(inBuf[lineNum]))
            outBuf.append("// [Blank Page]")
            logging.debug("{:>{:d}}: '{}' to '{}'".format(str(lineNum+1), len(str(len(inBuf))), inBuf[lineNum], outBuf[-1]))
            lineNum += 1
            count += 1

        else:
            outBuf.append(inBuf[lineNum])
            lineNum += 1

    logging.info("Processed {} blank pages".format(count))

    return outBuf


# Replace : -----File: 001.png---\sparkleshine\swankypup\Kipling\SeaRose\Scholar\------
# with    : // 001.png
def processPageNumbers(inBuf, keepOriginal):
    outBuf = []
    lineNum = 0
    count = 0

    logging.info("Processing page numbers")

    while lineNum < len(inBuf):
        if isLinePageBreak(inBuf[lineNum]):
            scanPageNum = parseScanPage(inBuf[lineNum])
            if keepOriginal:
                outBuf.append("// *** DP2PPGEN ORIGINAL: {}".format(inBuf[lineNum]))
            s = ".bn {0} // -----------------------({0})".format(scanPageNum)
            outBuf.append("{0}{1}".format(s, '-'*max(72-len(s), 0)))
            outBuf.append(".pn +1")
            logging.debug("{}: Page {}".format(str(lineNum+1), scanPageNum))
            lineNum += 1
            count += 1

        else:
            outBuf.append(inBuf[lineNum])
            lineNum += 1

    logging.info("Processed {} page numbers".format(count))

    return outBuf

def getDpMarkupBlock(buf, startLine):
    #TODO return line(s) containing /* */ /# #/ [] block
    return

def isNextOriginalLineBlank(buf, startLine):
    result = None
    lineNum = startLine
    while lineNum < len(buf) and result is None:
        if isLineBlank(buf[lineNum]):
            result = True
        elif isLineOriginalText(buf[lineNum]):
            result = False
        lineNum += 1

    return result

def isPreviousOriginalLineBlank(buf, startLine):
    result = None
    lineNum = startLine
    while lineNum >= 0 and result is None:
        if isLineBlank(buf[lineNum]):
            result = True
        elif isLineOriginalText(buf[lineNum]):
            result = False
        lineNum -= 1

    return result

def isLineBlank(line):
    return re.match(r"\s*$", line)

def isLineComment(line):
    return line.startswith("//")

def isLinePageBreak(line):
    return (parseScanPage(line) is not None)

def isDotCommand(line):
    return re.match(r"\.[a-z0-9]{2}[ -]", line)

def isLineOriginalText(line):
    # Non-original lines are:
    # ppgen dot commands
    # ppgen comment
    # dp proofing markup
    #   dp out of line formatting markup
    #   dp [] style markup (Illustration, footnote, sidenote..)
    return not re.match(r"(\.[a-z0-9]{2} |[*#]\/|\/[*#]|\*?\[\w+|\/\/)", line) and not isLinePageBreak(line)


def parseScanPage(line):
    scanPageNum = None

    m = re.match(r"-----File: (\w+\.(png|jpg|jpeg)).*", line)
    if m:
        scanPageNum = m.group(1)

    m = re.match(r"\/\/ (\w+\.(png|jpg|jpeg))", line)
    if m:
        scanPageNum = m.group(1)

    m = re.match(r"\.bn (\w+\.(png|jpg|jpeg))", line)
    if m:
        scanPageNum = m.group(1)

    return scanPageNum


def formatAsID(s):
    s = re.sub(r"<\/?\w+>", "", s)  # Remove inline markup
    s = re.sub(r"[^A-Za-z0-9_ ]", "", s)   # Strip everything but alphanumeric and _
    s = re.sub(r" ", "_", s.rstrip())        # Replace spaces with underscore
    s = s.lower()                   # Lowercase

    return s


def findNextEmptyLine(buf, startLine):
    lineNum = startLine
    retVal = None
    while lineNum < len(buf) and not retVal:
        if isLineBlank(buf[lineNum]):
            retVal = lineNum
        lineNum += 1
    return retVal

def findPreviousEmptyLine(buf, startLine):
    lineNum = startLine
    retVal = None
    while lineNum >= 0 and not retVal:
        if isLineBlank(buf[lineNum]):
            retVal = lineNum
        lineNum -= 1
    return retVal

def findNextNonEmptyLine(buf, startLine):
    lineNum = startLine
    retVal = None
    while lineNum < len(buf) and not retVal:
        if not isLineBlank(buf[lineNum]):
            retVal = lineNum
        lineNum += 1
    return retVal

def findPreviousNonEmptyLine(buf, startLine):
    lineNum = startLine
    retVal = None
    while lineNum >= 0 and not retVal:
        if not isLineBlank(buf[lineNum]):
            retVal = lineNum
        lineNum -= 1
    return retVal

# find previous line that contains original book text
# (ignore ppgen markup, proofing markup, blank lines)
def findPreviousLineOfText(buf, startLine):
    lineNum = findPreviousNonEmptyLine(buf, startLine)
    retVal = None
    while lineNum >= 0 and not retVal:
        if isLineOriginalText(buf[lineNum]):
            retVal = lineNum
        elif lineNum > 0:
            lineNum = findPreviousNonEmptyLine(buf, lineNum-1)
    return lineNum

# find next line that contains original book text
# (ignore ppgen markup, proofing markup, blank lines)
def findNextLineOfText(buf, startLine):
    lineNum = findNextNonEmptyLine(buf, startLine)
    retVal = None
    while lineNum < len(buf) and not retVal:
        if isLineOriginalText(buf[lineNum]):
            retVal = lineNum
        elif lineNum < len(buf)-1:
            lineNum = findNextNonEmptyLine(buf, lineNum+1)
    return lineNum

def findNextChapter(buf, startLine):
    lineNum = startLine
    retVal = None
    while lineNum < len(buf)-1 and not retVal:
        if buf[lineNum].startswith(".h2"):
            retVal = lineNum
        lineNum += 1
    return retVal


def processHeadings(inBuf, doChapterHeadings, doSectionHeadings, keepOriginal, chapterMaxLines, sectionMaxLines):
    outBuf = []
    lineNum = 0
    consecutiveEmptyLineCount = 0
    rewrapLevel = 0
    foundChapterHeadingStart = False
    chapterCount = 0
    sectionCount = 0

    if doChapterHeadings and doSectionHeadings:
        logging.info("Processing chapter and section headings")
    elif doChapterHeadings:
        logging.info("Processing chapter headings")
    elif doSectionHeadings:
        logging.info("Processing section headings")

    while lineNum < len(inBuf):
        # Chapter heading blocks are in the form:
            # (4 empty lines)
            # chapter name
            # can span more than one line
            # (1 empty line)
            # chapter description, opening quote, etc., 1 empty line seperating each
            # ...
            # (2 empty lines)

        # Section heading blocks are in the form
            # (2 empty lines)
            # section name
            # can span more than one line
            # (1 empty line)

        # Detect when inside out-of-line formatting block /# #/ /* */
        if inBuf[lineNum].startswith("/*") or inBuf[lineNum].startswith("/#"):
            rewrapLevel += 1
        elif inBuf[lineNum].startswith("*/") or inBuf[lineNum].startswith("#/"):
            rewrapLevel -= 1

        # Chapter heading
        if (doChapterHeadings and
                consecutiveEmptyLineCount == 4 and
                not isLineBlank(inBuf[lineNum]) and
                rewrapLevel == 0):
            inBlock = []
            outBlock = []
            foundChapterHeadingEnd = False
            consecutiveEmptyLineCount = 0

            # Copy chapter heading block to inBlock
            while lineNum < len(inBuf) and not foundChapterHeadingEnd:
                if isLineBlank(inBuf[lineNum]):
                    consecutiveEmptyLineCount += 1
                    if consecutiveEmptyLineCount == 2:
                        foundChapterHeadingEnd = True
                        consecutiveEmptyLineCount = 0
                else:
                    consecutiveEmptyLineCount = 0

                # chapters don't span pages
                if isLinePageBreak(inBuf[lineNum]):
                    foundChapterHeadingEnd = True

                if foundChapterHeadingEnd:
                    # Remove empty lines from end of chapter heading block
                    while isLineBlank(inBlock[-1]):
                        inBlock = inBlock[:-1]
                    # Set consecutiveEmptyLineCount to account for blank line removed (to handle back to back chapter headings)
                    consecutiveEmptyLineCount = 1
                else:
                    inBlock.append(inBuf[lineNum])
                    lineNum += 1

            # .sp 4
            # .h2 id=chapter_vi
            # CHAPTER VI.||chapter description etc..
            # .sp 2

            # Join chapter lines into one line
            chapterID = formatAsID(inBlock[0])
            chapterLine = ""
            extra = []
            doneChapterLine = False
            for line in inBlock:
                if not isLineOriginalText(line):
                    doneChapterLine = True

                if not doneChapterLine:
                    chapterLine += line
                    chapterLine += "|"
                else:
                    extra.append(line)

            if not chapterLine:
                logging.warning("Line {}: Disregarding chapter heading; no text found\n         {}".format(lineNum+1, inBlock[0]))
                for line in inBlock:
                    outBuf.append(line)
            elif len(inBlock) > int(chapterMaxLines):
                logging.warning("Line {}: Disregarding chapter heading; too many lines ({} > {}):\n ---\n{}\n ---".format((lineNum-len(inBlock))+1, len(inBlock), chapterMaxLines, "\n".join(inBlock[0:6])))
                for line in inBlock:
                    outBuf.append(line)

            else:
                while chapterLine[-1] == "|":
                    chapterLine = chapterLine[:-1]

                outBlock.append("")
                if keepOriginal:
                    outBlock.append("// ******** DP2PPGEN GENERATED ****************************************")
                outBlock.append(".sp 4")
                outBlock.append(".h2 id={}".format(chapterID))
                outBlock.append(chapterLine)
                outBlock.extend(extra)
                outBlock.append(".sp 2")

                if keepOriginal:
                    # Write out original as a comment
                    outBlock.append(".ig  // *** DP2PPGEN BEGIN ORIGINAL ***********************************")
                    outBlock.append("")
                    outBlock.append("")
                    outBlock.append("")
                    for line in inBlock:
                        outBlock.append(line)
                    outBlock.append("")
                    outBlock.append(".ig- // *** END *****************************************************")

                # Remove the consecutive blank lines that preceed chapter heading
                while isLineBlank(outBuf[-1]):
                    outBuf = outBuf[:-1]

                # Write out chapter heading block
                for line in outBlock:
                    outBuf.append(line)

                # Log action
                logging.info("-- .h2 {}".format(chapterLine))
                chapterCount += 1

        # Section heading
        elif doSectionHeadings and consecutiveEmptyLineCount == 2 and not isLineBlank(inBuf[lineNum]) and rewrapLevel == 0:
            inBlock = []
            outBlock = []
            foundSectionHeadingEnd = False
            consecutiveEmptyLineCount = 0

            # Copy section heading block to inBlock
            while lineNum < len(inBuf) and not foundSectionHeadingEnd:
                if isLineBlank(inBuf[lineNum]):
                    foundSectionHeadingEnd = True
                else:
                    inBlock.append(inBuf[lineNum])
                    lineNum += 1

            # TODO: Join section lines into one line.. really needed?

            # Check if this is a heading
            if len(inBlock) > int(sectionMaxLines):
                logging.debug("Line {}: Disregarding section heading; too many lines ({} > {}):\n ---\n{}\n ---".format((lineNum-len(inBlock))+1, len(inBlock), sectionMaxLines, "\n".join(inBlock[0:6])))
                for line in inBlock:
                    outBuf.append(line)
            else:
                # Remove one of the two consecutive blank lines that preceed section heading
                outBuf = outBuf[:-1]

                # .sp 2
                # .h3 id=section_i
                # Section I.
                # .sp 1

                sectionID = formatAsID(inBlock[0])
                sectionLine = ""
                for line in inBlock:
                    sectionLine += line
                    sectionLine += "|"
                sectionLine = sectionLine[:-1]

                if keepOriginal:
                    outBlock.append("// ******** DP2PPGEN GENERATED ****************************************")
                outBlock.append(".sp 2")
                outBlock.append(".h3 id={}".format(sectionID))
                outBlock.append(sectionLine)
                outBlock.append(".sp 1")

                if keepOriginal:
                    # Write out original as a comment
                    outBlock.append(".ig  // *** DP2PPGEN BEGIN ORIGINAL ***********************************")
                    outBlock.append("")
                    for line in inBlock:
                        outBlock.append(line)
                    outBlock.append(".ig- // *** END *****************************************************")

                # Write out chapter heading block
                for line in outBlock:
                    outBuf.append(line)

                # Log action
                logging.info("---- .h3 {}".format(inBlock[0]))
                sectionCount += 1

        else:
            if isLineBlank(inBuf[lineNum]):
                consecutiveEmptyLineCount += 1
            else:
                consecutiveEmptyLineCount = 0

            outBuf.append(inBuf[lineNum])
            lineNum += 1

    if doChapterHeadings:
        logging.info("Processed {} chapters".format(chapterCount))

    if doSectionHeadings:
        logging.info("Processed {} sections".format(sectionCount))

    return outBuf


def detectMarkup(inBuf):
    outBuf = []
    lineNum = 0
    rewrapLevel = 0
    markupCount = {'nf':0, 'ta':0, 'table':0, 'toc':0, 'title':0, 'poetry':0, 'index':0, 'bq':0, 'hang':0 }

    while lineNum < len(inBuf):

        # Process no-wrap /* */ markup
        #   table
        #   toc
        #   titlepage
        #   poetry
        #   index
        m = re.match(r"\/(\*|\#)(.*)", inBuf[lineNum])
        if m:
            inBlock = []
            outBlock = []
            foundChapterHeadingEnd = False
            consecutiveEmptyLineCount = 0
            markupType = m.group(2).split(" ")[0]
            dpType = m.group(1)

            # Copy nowrap block
            #TODO handle nested case where /* /* */ */
            lineNum += 1
            while lineNum < len(inBuf) and not inBuf[lineNum].startswith("{}/".format(dpType)):
                inBlock.append(inBuf[lineNum])
                lineNum += 1
            lineNum += 1

            # autodetect markup
            if not markupType:
                markupType = detectMarkupType(inBlock, dpType)

            if markupType:
                markupCount[markupType] += 1

            outBuf.append("/{}{}".format(dpType, markupType))
            for line in inBlock:
                outBuf.append(line)
            outBuf.append("{}/".format(dpType))

        else:
            outBuf.append(inBuf[lineNum])
            lineNum += 1

    return outBuf


def processOOLFMarkup(inBuf, keepOriginal):
    outBuf = []
    lineNum = 0
    rewrapLevel = 0
    markupCount = {'nf':0, 'ta':0, 'table':0, 'toc':0, 'title':0, 'poetry':0, 'index':0, 'bq':0, 'hang':0, 'signature':0 }

    while lineNum < len(inBuf):

        # Process no-wrap /* */ markup
        #   table
        #   toc
        #   titlepage
        #   poetry
        #   index
        m = re.match(r"\/(\*|\#)(.*)", inBuf[lineNum])
        if m:
            inBlock = []
            outBlock = []
            foundChapterHeadingEnd = False
            consecutiveEmptyLineCount = 0
            markupType = parseMarkupType(m.group(2))
            args = parseArgs(m.group(2))
            dpType = m.group(1)

            # Copy nowrap block
            #TODO handle nested case where /* /* */ */
            nowrapStartLine = inBuf[lineNum]
            lineNum += 1
            while lineNum < len(inBuf) and not inBuf[lineNum].startswith("{}/".format(dpType)):
                inBlock.append(inBuf[lineNum])
                lineNum += 1
            nowrapEndLine = inBuf[lineNum]
            lineNum += 1

            if markupType:
                logging.info("----- Found {}, line {}".format(markupType, lineNum))

                if markupType == "nf":
                    outBlock = processNf(inBlock, keepOriginal, args)
                elif markupType == "ta":
                    outBlock = processTa(inBlock, keepOriginal, args)
                elif markupType == "table":
                    outBlock = processTable(inBlock, keepOriginal)
                elif markupType == "toc":
                    outBlock = processToc(inBlock, keepOriginal, args)
                elif markupType == "title":
                    outBlock = processTitlePage(inBlock, keepOriginal)
                elif markupType == "poetry":
                    outBlock = processPoetry(inBlock, keepOriginal)
                elif markupType == "index":
                    outBlock = processIndex(inBlock, keepOriginal, args)
                elif markupType == "bq":
                    outBlock = processBlockquote(inBlock, keepOriginal, args)
                elif markupType == "hang":
                    outBlock = processHangingIndent(inBlock, keepOriginal, args)
                elif markupType == "signature":
                    outBlock = processSignature(inBlock, keepOriginal, args)
                else:
                    logging.warn("{}: Unknown markup type '{}' found".format(lineNum+1, markupType))

                c = markupCount.get(markupType, 0)
                markupCount[markupType] = c + 1

                for line in outBlock:
                    outBuf.append(line)
            else:
                outBuf.append(nowrapStartLine)
                for line in inBlock:
                    outBuf.append(line)
                outBuf.append(nowrapEndLine)

        else:
            outBuf.append(inBuf[lineNum])
            lineNum += 1

    # Add table CSS
    if markupCount['table'] > 0:
        cssBlock = []
        cssBlock.append("// Tables")
        cssBlock.append(".de .tableU1 { page-break-inside: avoid; margin: 1.5em auto; border-collapse: collapse; width: auto; max-width: 97%}")
        cssBlock.append(".de .tableU1 td, .tableU1 th { padding: 0.15em 0.5em; border-left: 1px solid black; border-right: 1px solid black; border-bottom: 1px solid black; text-align: center; font-size: small; }")
        cssBlock.append(".de .tableU1 th { padding: 0.8em 0.5em; font-weight: normal; font-size: smaller; border: 1px solid black; }")
        cssBlock.append(".de .tableU1 td div.lgcurly { font-size:300%;font-weight:lighter;margin:0;line-height:1em;text-indent:0; }")
        cssBlock.append("")
        cssBlock.append(".de caption { margin-bottom: 0.8em; font-weight: bold; font-size: 0.9em; }")
        cssBlock.append(".de td.ybt, th.ybt { border-top: 1px solid black; }")
        cssBlock.append(".de td.nbt, th.nbt { border-top-style: none; }")
        cssBlock.append(".de td.nbb, th.nbb { border-bottom-style: none; }")
        cssBlock.append(".de td.nbl, th.nbl { border-left-style: none; }")
        cssBlock.append(".de td.nbr, th.nbr { border-right-style: none; }")
        cssBlock.append(".de td.dbt, th.dbt { border-top: double; }")
        cssBlock.append(".de td.dbb, th.dbb { border-bottom: double; }")
        cssBlock.append(".de td.dbr, th.dbr { border-right: double; }")
        cssBlock.append(".de td.valignb, th.valignb { vertical-align: bottom; }")
        cssBlock.append(".de td.left { text-align: left; }")
        cssBlock.append(".de td.right { text-align: right; }")
        cssBlock.append(".de td.hang { text-align: left; padding-left: 1.5em; text-indent: -1.2em; }")
        cssBlock.append(".de td.hang2 { text-align: left; padding-left: 3em; text-indent: -1.2em; }")
        cssBlock.append(".de .nodecoration { text-decoration: none; }")
        cssBlock.append("")

        outBuf[0:0] = cssBlock

    return outBuf


def processNf(inBuf, keepOriginal, args):
    outBuf = []
    lineNum = 0

    # Use arguments if provided
    align = 'c'
    if 'c' in args:
        align = 'c'
    if 'l' in args:
        align = 'l'
    if 'r' in args:
        align = 'r'

    outBuf.append(".nf {}".format(align))

    while lineNum < len(inBuf):
        outBuf.append(inBuf[lineNum])
        lineNum += 1

    outBuf.append(".nf-")

    return outBuf


def processTa(inBuf, keepOriginal, args):
    outBuf = []
    lineNum = 0

    # Use arguments if provided
    columns = ""
    if 'columns' in args:
        columns = args['columns']
    s = ""
    if 's' in args:
        s = args['s']
    r = ""
    if 'r' in args:
        r = args['r']

    outBuf.append(".ta {}".format(columns))

    while lineNum < len(inBuf):
        m = re.search(s, inBuf[lineNum])
        if m:
            print("{}: {}".format(lineNum+1, inBuf[lineNum]))

        if isLineOriginalText(inBuf[lineNum]):
            inBuf[lineNum] = re.sub(s, r, inBuf[lineNum])
        outBuf.append(inBuf[lineNum])
        lineNum += 1

    outBuf.append(".ta-")

    return outBuf


def processTitlePage(inBuf, keepOriginal):
    outBuf = []
    lineNum = 0

    outBuf.append(".nf c")

    while lineNum < len(inBuf):
        # TODO
        # H1 from first line until blank line
        # <l> markup everything else
        outBuf.append(inBuf[lineNum])
        lineNum += 1

    outBuf.append(".nf-")

    return outBuf


def processBlockquote(inBuf, keepOriginal, args):
    outBuf = []
    lineNum = 0

    # Use arguments if provided
    indent = 2
    if 'in' in args:
        indent = args['in']

    outBuf.append(".in {}".format(indent))
    outBuf.append(".ll -{}".format(indent))

    while lineNum < len(inBuf):
        outBuf.append(inBuf[lineNum])
        lineNum += 1

    outBuf.append(".ll")
    outBuf.append(".in")

    return outBuf


def processSignature(inBuf, keepOriginal, args):
    outBuf = []
    lineNum = 0

    # Use arguments if provided
    indent = 2
    if 'in' in args:
        indent = args['in']

    outBuf.append(".sp 1".format(indent))
    outBuf.append(".in +{}".format(indent))
    outBuf.append(".ll -{}".format(indent))
    outBuf.append(".rj {}".format(len(inBuf)))

    while lineNum < len(inBuf):
        outBuf.append(inBuf[lineNum])
        lineNum += 1

    outBuf.append(".ll")
    outBuf.append(".in")

    return outBuf


def processHangingIndent(inBuf, keepOriginal, args):
    outBuf = []
    lineNum = 0

    # Use arguments if provided
    indent = 4
    if 'in' in args:
        indent = args['in']

    outBuf.append(".in {}".format(indent))
    outBuf.append(".ti -{}".format(indent))

    while lineNum < len(inBuf):
        outBuf.append(inBuf[lineNum])
        lineNum += 1

    outBuf.append(".in")

    return outBuf


def processIndex(inBuf, keepOriginal, args):
    outBuf = []
    lineNum = 0

    # Use arguments if provided
    s = r", (\d{1,3})(?!\d)"
    if 's' in args:
        s = args['s']
    r = r", #\1#"
    if 'r' in args:
        r = args['r']
    indent = 2
    if 'in' in args:
        indent = args['in']

    outBuf.append(".na")
    outBuf.append(".in {}".format(indent))
    outBuf.append(".nf l")

    while lineNum < len(inBuf):
        if isLineOriginalText(inBuf[lineNum]):
            m = re.search(r"(\d{4,})", inBuf[lineNum])
            if m:
                logging.warning("Link not created for digit in index: {}".format(m.group(1)))
            inBuf[lineNum] = re.sub(s, r, inBuf[lineNum])
        outBuf.append(inBuf[lineNum])
        lineNum += 1

    outBuf.append(".nf-")
    outBuf.append(".in")
    outBuf.append(".ad")

    return outBuf


# For a given ppgen command, parse all arguments in the form
#   arg="val"
#   arg='val'
#   arg=val
#   val val
def parseArgs(commandLine):
    args = {}

    # break up command line
    tokens = shlex.split(commandLine)
    #print(tokens)

    # process parameters (skip .command)
    for t in tokens[1:]:
        t = re.sub(r"[\'\"]", "", t)
        m = re.match(r"(.+)=(.+)", t)
        if m:
            args[m.group(1)]=m.group(2)
        else:
            args[t]=t

    return(args)


def parseMarkupType(commandLine):
    markupType = commandLine.split(" ")[0]

    if markupType == "sig":
        markupType == "signature"
    elif markupType == "poem":
        markupType == "poetry"

    return(markupType)


def processToc(inBuf, keepOriginal, args):
    outBuf = []
    lineNum = 0

    tocStyles = (
        # 3. County and Shire. Meaning of the Words      42
        { 's': r'^(\d+?\.) (.+?) {6,}(\d+)', 'r': r'\1|#\2:Page_\3#|#\3#', 'count': 0, 'columns': 'rlr' },
        # XI. Columbus and the Savages      48
        { 's': r'^([XIVLC]+?\.) (.+?) {6,}(\d+)', 'r': r'\1|#\2:Page_\3#|#\3#', 'count': 0, 'columns': 'rlr' },
        # SIR CHRISTOPHER WREN      24
        { 's': r'^(.+?) {6,}(\d+)', 'r': r'#\1:Page_\2#|#\2#', 'count': 0, 'columns': 'lr' },
   )

    # Does toc fit a known style?
    for style in tocStyles:
        for line in inBuf:
            if re.search(style['s'], line):
                style['count'] += 1

    styleUsed = ""
    maxCount = 0
    for style in tocStyles:
        if style['count'] > 0:
            styleUsed = style
            break

    # Use arguments if provided
    columns = style['columns']
    if 'columns' in args:
        columns = args['columns']
    s = styleUsed['s']
    if 's' in args:
        s = args['s']
    r = styleUsed['r']
    if 'r' in args:
        r = args['r']

    outBuf.append(".ta {}".format(columns))

    while lineNum < len(inBuf):
        m = re.search(s, inBuf[lineNum])
        if m:
            print("{}: {}".format(lineNum+1, inBuf[lineNum]))

        if isLineOriginalText(inBuf[lineNum]):
            inBuf[lineNum] = re.sub(s, r, inBuf[lineNum])
        outBuf.append(inBuf[lineNum])
        lineNum += 1

    outBuf.append(".ta-")

    return outBuf


def processPoetry(inBuf, keepOriginal):
    outBuf = []
    lineNum = 0

    outBuf.append(".nf b")

    while lineNum < len(inBuf):
        outBuf.append(inBuf[lineNum])
        lineNum += 1

    outBuf.append(".nf-")

    return outBuf


def processTable(inBuf, keepOriginal):
    outBuf = []
    lineNum = 0

    for line in inBuf:
        logging.info(line)

    # Correct markup for rst, warn about things that need manual intervention
    rstBlock = dpTableToRst(inBuf)

    # Run through rst2html
    logging.info("\n----- Generating HTML with rst2html")
    tableHTML = rstTableToHTML(rstBlock)

    # Build ppgen code
    outBuf.append(".if t")
    outBuf.append(".nf b")
    for line in inBuf:
        outBuf.append(line)
    outBuf.append(".nf-")
    outBuf.append(".if-")
    outBuf.append(".if h")
    outBuf.append(".li")
    for line in tableHTML:
        outBuf.append(line)
    outBuf.append(".li-")
    outBuf.append(".if-")

    return outBuf


def rstTableToHTML(inBuf):

    # Build input to rstToHtml
    inFile = tempfile.NamedTemporaryFile(delete=False)
    inFileName=inFile.name
    for line in inBuf:
        inFile.write(bytes(line+'\n', 'UTF-8'))
    inFile.close()

    # Process table with rst2html
    outFileName=makeTempFile()
    commandLine=['rst2html', inFileName, outFileName]
    logging.debug("commandLine:{}".format(str(commandLine)))
    proc=subprocess.Popen(commandLine)
    proc.wait()
    if(proc.returncode != 0):
        logging.error("Command failed: {}".format(str(commandLine)))

    # Parse table HTML from rst2html output
    outBuf = []
    inTable = False
    for line in loadFile(outFileName):
        if "<table" in line:
            line = '<table class="tableU1">'
            inTable = True

        if "</table" in line:
            outBuf.append(line)
            inTable = False

        if inTable:
            outBuf.append(line)

    # Compact rows, strip colgroup
    inBuf = outBuf[:]
    outBuf = []
    inTr = False
    inColgroup = False
    rowStart = 0
    for i, line in enumerate(inBuf):
        if "<colgroup" in line:
            inColgroup = True
        elif "<tr" in line:
            inTr = True
            rowStart = i

        if "</tr" in line:
            rowHTML = ' '.join(inBuf[rowStart:i])
            outBuf.append(rowHTML)
            inTr = False
        if "</colgroup" in line:
            inColgroup = False
        elif not inTr and not inColgroup:
            outBuf.append(line)

    # Strip tbody
    inBuf = outBuf[:]
    outBuf = []
    for i, line in enumerate(inBuf):
        if not re.match("</?tbody", line):
            outBuf.append(line)

    # Assume first row is header row
    done = False
    for i, line in enumerate(outBuf):
        if "<tr" in line and not done:
            outBuf[i] = outBuf[i].replace("<td", "<th")
            outBuf[i] = outBuf[i].replace("</td>", "</th>")
            done = True

    return outBuf


def makeTempFile():
    tf = tempfile.NamedTemporaryFile(delete=False)
    fn = tf.name
    tf.close()
    return fn


def dpTableToRst(inBuf):
    outBuf = inBuf[:]
    tableWidth = 0

    # Trim whitespace
    for i, line in enumerate(outBuf):
        outBuf[i] = outBuf[i].rstrip()

    # Add left/right edges if needed
    inTable = False
    for i, line in enumerate(outBuf):

        if re.match(r"\+[-=]", line):
            inTable = True
        elif re.match(r"[-=]", line):
            outBuf[i] = "+{}".format(outBuf[i])
            inTable = True
        elif re.match(r"[^|+]", line) and inTable:
            outBuf[i] = "|{}".format(outBuf[i])
        if re.search(r"[-=]$", line):
            outBuf[i] = "{}+".format(outBuf[i])
            tableWidth = len(outBuf[i])
        elif re.search(r"[^|+]$", line) and inTable or tableWidth > len(outBuf[i]):
            if tableWidth > len(outBuf[i]):
                s = "{0:<{tableWidth}}|".format(outBuf[i], tableWidth=(tableWidth-1))
                outBuf[i] = s
            else:
                outBuf[i] = "{}|".format(outBuf[i])

        # Left align cell text
        m = re.findall(r"\|([^|+]+)", outBuf[i])
        for cell in m:
            cw = len(cell)
            if re.search(r"[^|\s]", cell):
                s = r"|{}".format(cell)
                r = r"|{0:<{cw}}".format(cell.lstrip(), cw=cw)
                outBuf[i] = outBuf[i].replace(s, r)

        # Ignore lines not inside table (title etc.)
        if not inTable and line != "":
            logging.warn("Ignoring line outside table:\n{}".format(line))
            del outBuf[i]

    return outBuf


def detectMarkupType(buf, dpType):
    # Tables
    matches = {
              "table": {
                       "--------+": 0,
                       "=========": 0,
                       "---------": 0,
                       #TODO update and test better markup
                       # "[-=]{6,}+": False,
                       # "[-=]{6,}": False,
                       "|": 0,
                       "T[aAbBlLeE]": 0
              },
              "toc": {
                       " {6,}\d+": 0,
              },
              "titlepage": {
                       "": 0,
                       #TODO.. close to top? is autodetect possible?
              }
    }

    for line in buf:
        for key in matches:
            for s in matches[key]:
                if re.search(s, line) and isLineOriginalText(line):
                    matches[key][s] += 1

    if (matches["table"]["--------+"] and matches["table"]["|"]) or (matches["table"]["T[aAbBlLeE]"] and matches["table"]["--------+"]):
        return "table"
    elif matches["toc"][" {6,}\d+"]:
        return "toc"

    return ""


def fatal(errorMsg):
    logging.critical(errorMsg)
    exit(1)
    return


def loadFile(fn):
    inBuf = []
    encoding = ""

    if not os.path.isfile(fn):
        fatal("File not found: {}".format(fn))

    if encoding == "":
        try:
            wbuf = open(fn, "r", encoding='ascii').read()
            encoding = "ASCII" # we consider ASCII as a subset of Latin-1 for DP purposes
            inBuf = wbuf.split("\n")
        except Exception as e:
            pass

    if encoding == "":
        try:
            wbuf = open(fn, "rU", encoding='UTF-8').read()
            encoding = "utf_8"
            inBuf = wbuf.split("\n")
            # remove BOM on first line if present
            t = ":".join("{0:x}".format(ord(c)) for c in inBuf[0])
            if t[0:4] == 'feff':
                inBuf[0] = inBuf[0][1:]
        except:
            pass

    if encoding == "":
        try:
            wbuf = open(fn, "r", encoding='latin_1').read()
            encoding = "latin_1"
            inBuf = wbuf.split("\n")
        except Exception as e:
            pass

    if encoding == "":
        fatal("Cannot determine input file decoding")
    else:
        # self.info("input file is: {}".format(encoding))
        if encoding == "ASCII":
            encoding = "latin_1" # handle ASCII as Latin-1 for DP purposes

    for i in range(len(inBuf)):
        inBuf[i] = inBuf[i].rstrip()

    return inBuf


def createOutputFileName(infile):
    # TODO make this smart.. is infile raw or ppgen source? maybe two functions needed
    outfile = "{}-out.txt".format(infile.split('.')[0])
    return outfile

def stripFootnoteMarkup(inBuf):
    outBuf = []
    lineNum = 0

    while lineNum < len(inBuf):
        # copy inBuf to outBuf throwing away all footnote markup [Footnote...]
        if re.match(r"\*?\[Footnote", inBuf[lineNum]):
            bracketLevel = 0
            done = False
            while lineNum < len(inBuf)-1 and not done:
                # Detect when [ ] level so that nested [] are handled properly
                m = re.findall("\[", inBuf[lineNum])
                for b in m:
                    bracketLevel += 1
                m = re.findall("\]", inBuf[lineNum])
                for b in m:
                    bracketLevel -= 1

                if bracketLevel == 0 and re.search(r"][\*]*$", inBuf[lineNum]):
                    done = True
                    lineNum += 1
                else:
                    lineNum += 1
        else:
            outBuf.append(inBuf[lineNum])
            lineNum += 1

    return outBuf


def processSidenotes(inBuf, keepOriginal, keepBreaks):
    sidenotesCount = 0
    lineNum = 0
    outBuf = []

    logging.info("Processing sidenotes")
    while lineNum < len(inBuf):

        # Search for sidenotes
        if re.match(r"\*?\[Sidenote", inBuf[lineNum]):
            startLine = lineNum

            # Copy sidenote block
            snBlock = []
            snBlock.append(inBuf[lineNum])
            while lineNum < len(inBuf)-1 and not re.search(r"][\*]*$", inBuf[lineNum]):
                lineNum += 1
                snBlock.append(inBuf[lineNum])

            endLine = lineNum

            # Strip markup text from [sidenote] block
            snText = []
            for line in snBlock:
                line = re.sub(r"^\*?\[Sidenote: ?", "", line)
                line = re.sub(r"]$", "", line)
                snText.append(line)

            # Need to relocate *[Sidenote
            if inBuf[startLine][0] == '*':
                outBuf.append("// *** DP2PPGEN: RELOCATE SIDENOTE")

            # Ouput ppgen style sidenote
            if keepBreaks:
                joinChar = '|'
            else:
                joinChar = ' '
            s = ".sn {}".format(joinChar.join(snText))
            outBuf.append(s)
            sidenotesCount += 1
            lineNum += 1

        else:
            outBuf.append(inBuf[lineNum])
            lineNum += 1

    logging.info("Processed {} sidenotes".format(sidenotesCount))

    return outBuf


def parseFootnotes(inBuf):
# parse footnotes into a list of dictionaries with the following properties for each entry
#   startLine - line number of [Footnote start
#   endLine - line number of last line of [Footnote] block
#   fnBlock - list of lines containing full [Footnote:]
#   fnText - list of lines containing footnote text
#   paragraphEnd - line number of the blank line following the paragraph this footnote is located in
#   chapterEnd - line number of the blank line following the last paragraph in the chapter this footnote is located in
#   scanPageNumber - scan page this footnote is located on

    footnotes = []
    lineNum = 0
    currentScanPage = 0

    logging.info("-- Parsing footnotes")
    while lineNum < len(inBuf):
        foundFootnote = False

        # Keep track of active scanpage
        if isLinePageBreak(inBuf[lineNum]):
            currentScanPage = parseScanPage(inBuf[lineNum])

        if re.match(r"\*?\[Footnote", inBuf[lineNum]):
            foundFootnote = True

        if foundFootnote:
            startLine = lineNum

            # Copy footnote block
            fnBlock = []
            fnBlock.append(inBuf[lineNum])

            bracketLevel = 0
            done = False
            while lineNum < len(inBuf)-1 and not done:
                # Detect when [ ] level so that nested [] are handled properly
                m = re.findall("\[", inBuf[lineNum])
                for b in m:
                    bracketLevel += 1
                m = re.findall("\]", inBuf[lineNum])
                for b in m:
                    bracketLevel -= 1

                if bracketLevel == 0 and re.search(r"][\*]*$", inBuf[lineNum]):
                    done = True
                else:
                    lineNum += 1
                    fnBlock.append(inBuf[lineNum])

            endLine = lineNum

            # Is footnote part of a multipage footnote?
            joinToPrevious = fnBlock[0].startswith("*[Footnote")
            joinToNext = fnBlock[-1].endswith("]*")
            if joinToPrevious or joinToNext:
                logging.debug("Footnote requires joining at line {}: {}".format(lineNum+1, inBuf[lineNum]))
                foundFootnote = True

            # Find end of paragraph
            paragraphEnd = -1 # This must be done during footnote anchor processing as paragraph end is relative to anchor and not [Footnote] markup
            # Find end of chapter (line after last line of last paragraph)
            chapterEnd = -1 # This must be done during footnote anchor processing as chapter end is relative to anchor and not [Footnote] markup

            # Extract footnote ID
            m = re.match(r"\[Footnote (\w{1,2}):", fnBlock[0])
            if m:
                fnID = m.group(1)

            # Strip markup text from [Footnote] block
            fnText = []
            for line in fnBlock:
                line = re.sub(r"^\*\[Footnote: ?", "", line)
                line = re.sub(r"^\[Footnote [A-Za-z]: ?", "", line)
                line = re.sub(r"^\[Footnote \d+: ?", "", line)
                fnText.append(line)
            fnText[-1] = re.sub(r"][\*]*$", "", fnText[-1])

            # Add entry
            footnotes.append({'fnBlock':fnBlock, 'fnText':fnText, 'fnID':fnID, 'startLine':startLine, 'endLine':endLine, 'paragraphEnd':paragraphEnd, 'chapterEnd':chapterEnd, 'joinToPrevious':joinToPrevious, 'joinToNext':joinToNext, 'scanPageNum':currentScanPage})

        lineNum += 1

    logging.info("-- Parsed {} footnotes".format(len(footnotes)))

    # Join footnotes marked above during parsing
    joinCount = 0
    i = len(footnotes) - 1
    while i > 0:
        if footnotes[i]['joinToNext']:
            logging.error("Unresolved join detected")
            logging.error("ScanPg {} Footnote {} ({}): {}".format(footnotes[i]['scanPageNum'], i, footnotes[i]['startLine']+1, footnotes[i]['fnBlock'][0]))

        if footnotes[i]['joinToPrevious']:
            if joinCount == 0:
                logging.info("-- Joining footnotes")

            # Find footnote on previous page with joinToNext set
            toFn = i-1
            toScanPageNum = footnotes[toFn]['scanPageNum']
            while footnotes[toFn]['joinToNext']==False and footnotes[toFn]['scanPageNum']==toScanPageNum:
                toFn -= 1

            # debug message
            logging.debug("Merging footnote [{}]".format(i+1))
            if len(footnotes[toFn]['fnBlock']) > 1:
                logging.debug("  ScanPg {}: {} ... {} ".format(footnotes[toFn]['scanPageNum'], footnotes[toFn]['fnBlock'][0], footnotes[toFn]['fnBlock'][-1]))
            else:
                logging.debug("  ScanPg {}: {}".format(footnotes[toFn]['scanPageNum'], footnotes[toFn]['fnBlock'][0]))
            if len(footnotes[i]['fnBlock']) > 1:
                logging.debug("  ScanPg {}: {} ... {} ".format(footnotes[i]['scanPageNum'], footnotes[i]['fnBlock'][0], footnotes[i]['fnBlock'][-1]))
            else:
                logging.debug("  ScanPg {}: {}".format(footnotes[i]['scanPageNum'], footnotes[i]['fnBlock'][0]))

            if not footnotes[toFn]['joinToNext']:
                logging.error("Attempt to join footnote failed!")
                logging.error("ScanPg {} Footnote {} ({}): {}".format(footnotes[toFn]['scanPageNum'], i+1, footnotes[toFn]['startLine']+1, footnotes[toFn]['fnBlock'][0]))
                logging.error("ScanPg {} Footnote {} ({}): {}".format(footnotes[i]['scanPageNum'], i, footnotes[i]['startLine']+1, footnotes[i]['fnBlock'][0]))
            else:
                # handle spanned hyphenation within spanned footnote
                needsHyphenJoin = False
                if re.search(r"(?<![-—])-\*?$", footnotes[toFn]['fnText'][-1]):
                    if footnotes[i]['fnText'][0][0] != '*':
                        logging.error("Footnote {}: Unresolved hyphenation\n       {}\n       {}".format(toFn, footnotes[toFn]['fnText'][-1], footnotes[i]['fnText'][0][0]))
                    else:
                        needsHyphenJoin = True

                if needsHyphenJoin:
                    # Grab first word of from line
                    fromWord = footnotes[i]['fnText'][0].split(' ', 1)[0]
                    if len(footnotes[i]['fnText'][0].split(' ', 1)) > 1:
                        footnotes[i]['fnText'][0] = footnotes[i]['fnText'][0].split(' ', 1)[1]
                    else:
                        # Single word on from line, remove blank line
                        del footnotes[i]['fnText'][0]

                    # Append it to toline
                    footnotes[toFn]['fnText'][-1] = footnotes[toFn]['fnText'][-1] + fromWord

                # merge fnBlock and fnText from second into first
                footnotes[toFn]['fnBlock'].extend(footnotes[i]['fnBlock'])
                footnotes[toFn]['fnText'].extend(footnotes[i]['fnText'])
                footnotes[toFn]['joinToNext'] = False
                del footnotes[i]
                joinCount += 1

        i -= 1

    if joinCount > 0:
        logging.info("-- Merged {} broken footnote(s)".format(joinCount))
        logging.info("-- {} total footnotes after joining".format(len(footnotes)))

    return footnotes


def processFootnoteAnchors(inBuf, footnotes, useAutoNumbering):

    outBuf = inBuf

    # process footnote anchors
    fnUniqueAnchorCount = 0
    anchorCount = 0
    lineNum = 0
    currentScanPage = 0
    currentScanPageLabel = ""
    fnIDs = []
#   r = []
    logging.info("-- Processing footnote anchors")
    while lineNum < len(outBuf):

        # Keep track of active scanpage
        if isLinePageBreak(outBuf[lineNum]):
            anchorsThisPage = []
            currentScanPage = parseScanPage(inBuf[lineNum])

            # Make list of footnotes found on this page
            fnIDs = []
            for fn in footnotes:
                if fn['scanPageNum'] == currentScanPage:
                    fnIDs.append(fn['fnID'])

            # Build regex for footnote anchors that can be found on this scanpage
#           if fnIDs:
#               r = "|".join(fnIDs)
#               r = r"\[({})\]".format(r)

#       print("{}: {}".format(lineNum+1, outBuf[lineNum]))
        m = re.findall("\[([A-Za-z]|[0-9]{1,2})\]", outBuf[lineNum])
        for anchor in m:
            # Check that anchor found belongs to a footnote on this page
            if not anchor in fnIDs:
                logging.error("No matching footnote for anchor [{}] on scan page {} (line {} in output file):\n       {}".format(anchor, currentScanPage, lineNum+1, outBuf[lineNum]))
                logging.debug(fnIDs)

            else:
                # replace [1] or [A] with [n]
                curAnchor = "\[{}\]".format(anchor)
                if not curAnchor in anchorsThisPage:
                    fnUniqueAnchorCount += 1
                    anchorsThisPage.append(curAnchor)
                elif useAutoNumbering:
                    logging.error("Duplicate anchors ([{}]) detected ({}); ppgen autonumbering may not function correctly".format(anchor, currentScanPage))

                if useAutoNumbering:
                    newAnchor = "[#]"
                else:
                    newAnchor = "[{}]".format(fnUniqueAnchorCount)

                logging.debug("{:>5s}: ({}|{}) ... {} ...".format(newAnchor, lineNum+1, currentScanPageLabel, outBuf[lineNum]))
                for line in footnotes[fnUniqueAnchorCount-1]['fnText']:
                    logging.debug("       {}".format(line))

                # sanity check (anchor and footnote should be on same scan page)
                if currentScanPage != footnotes[fnUniqueAnchorCount-1]['scanPageNum']:
                    fatal("Anchor found on different scan page, anchor({}) and footnotes({}) may be out of sync".format(currentScanPage, footnotes[fnUniqueAnchorCount-1]['scanPageNum']))

                # replace anchor
                outBuf[lineNum] = re.sub(curAnchor, newAnchor, outBuf[lineNum])

                # update paragraphEnd and chapterEnd so they are relative to anchor and not [Footnote
                # Find end of paragraph
                paragraphEnd = findNextEmptyLine(outBuf, lineNum)
                footnotes[fnUniqueAnchorCount-1]['paragraphEnd'] = paragraphEnd

                # Find end of chapter (line after last line of last paragraph)
                # Chapter headings must be marked in ppgen format (.h2)
                chapterEnd = findNextChapter(outBuf, lineNum)
                if chapterEnd:
                    chapterEnd = findPreviousLineOfText(outBuf, chapterEnd) + 1
                    footnotes[fnUniqueAnchorCount-1]['chapterEnd'] = chapterEnd

        lineNum += 1

    logging.info("-- Processed {} footnote anchors".format(fnUniqueAnchorCount))

    return outBuf, fnUniqueAnchorCount


def processFootnotes(inBuf, footnoteDestination, keepOriginal, lzdestt, lzdesth, useAutoNumbering):
    outBuf = []

    logging.info("Processing footnotes")

    # strip empty lines before [Footnotes], *[Footnote
    lineNum = 0
    logging.info("-- Removing blank lines before [Footnotes]")
    while lineNum < len(inBuf):
        if re.match(r"\*?\[Footnote", inBuf[lineNum]):
            # delete previous blank line(s)
            while isLineBlank(outBuf[-1]):
                del outBuf[-1]

        outBuf.append(inBuf[lineNum])
        lineNum += 1
    inBuf = outBuf

    # parse footnotes into list of dictionaries
    footnotes = parseFootnotes(outBuf)

    outBuf = stripFootnoteMarkup(outBuf)

    # find and markup footnote anchors
    outBuf, fnUniqueAnchorCount = processFootnoteAnchors(outBuf, footnotes, useAutoNumbering)

    if len(footnotes) != fnUniqueAnchorCount:
        logging.error("Footnote anchor count does not match footnote count")

    if footnotes:
        outBuf = generatePpgenFootnoteMarkup(outBuf, footnotes, footnoteDestination, lzdestt, lzdesth, useAutoNumbering)
        if lzdestt or lzdesth:
            outBuf = generateLandingZones(outBuf, footnotes, lzdestt, lzdesth)

    logging.info("Processed {} footnotes".format(len(footnotes)))

    return outBuf


# Generate ppgen footnote markup
def generatePpgenFootnoteMarkup(inBuf, footnotes, footnoteDestination, lzdestt, lzdesth, useAutoNumbering):

    outBuf = inBuf

    logging.info("-- Generating footnote markup")

    fmText = ""
    if lzdestt and lzdesth:
        fmText = ".fm rend=no"
    elif lzdestt:
        fmText = ".fm rend=h"
    elif lzdesth:
        fmText = ".fm rend=t"
    else:
        fmText = ".fm"

    if footnoteDestination == "bookend":
        logging.info("-- Adding ppgen style footnotes to end of book")

        fnMarkup = []
        fnMarkup.append(".sp 4")
        fnMarkup.append(".pb")
        fnMarkup.append(".de div.footnotes { border: dashed 1px #aaaaaa; padding: 1.5em; }")
        fnMarkup.append(".de div.footnotes h2 { margin-top: 1em; }")
        fnMarkup.append('.dv class="footnotes"')
        fnMarkup.append(".sp 2")
        fnMarkup.append(".h2 id=footnotes nobreak")
        fnMarkup.append("FOOTNOTES:")
        fnMarkup.append(".sp 2")
        for i, fn in enumerate(footnotes):
            fnMarkup.append(".fn {}  // {}".format(i+1, fn['scanPageNum']))
            for line in fn['fnText']:
                fnMarkup.append(line)
            fnMarkup.append(".fn-")
        fnMarkup.append(".dv-")

        outBuf.extend(fnMarkup)

    elif footnoteDestination == "chapterend":
        logging.info("-- Adding ppgen style footnotes to end of chapters")
        curChapterEnd = footnotes[-1]['chapterEnd']
        for i, fn in reversed(list(enumerate(footnotes))):

            if curChapterEnd != fn['chapterEnd']:
                # finish off last group
                outBuf.insert(curChapterEnd, fmText)
                curChapterEnd = fn['chapterEnd']

            # build markup for this footnote
#           print("{} {}".format(fn['chapterEnd'], fn['fnText'][0]))
            fnMarkup = []
            if useAutoNumbering:
                fnMarkup.append(".fn #  // {}".format(fn['scanPageNum']))
            else:
                fnMarkup.append(".fn {}  // {}".format(i+1, fn['scanPageNum']))

            for line in fn['fnText']:
                fnMarkup.append(line)
            fnMarkup.append(".fn-")

            # insert it
            outBuf[curChapterEnd:curChapterEnd] = fnMarkup

        outBuf.insert(curChapterEnd, fmText)

    elif footnoteDestination == "paragraphend":
        logging.info("-- Adding ppgen style footnotes to end of paragraphs")
        curParagraphEnd = footnotes[-1]['paragraphEnd']
        for i, fn in reversed(list(enumerate(footnotes))):

            if curParagraphEnd != fn['paragraphEnd']:
                outBuf.insert(curParagraphEnd, fmText)
                curParagraphEnd = fn['paragraphEnd']

            # build markup for this footnote
#           print("{} {}".format(fn['paragraphEnd'], fn['fnText'][0]))
            fnMarkup = []
            fnMarkup.append(".fn {}".format(i+1))
            for line in fn['fnText']:
                fnMarkup.append(line)
            fnMarkup.append(".fn-")

            # insert it
            outBuf[curParagraphEnd:curParagraphEnd] = fnMarkup

        outBuf.insert(curParagraphEnd, fmText)

    else:
        logging.error("Unrecognized value for --fndest ({})".format(footnoteDestination))

    return outBuf


def generateLandingZones(inBuf, footnotes, lzdestt, lzdesth):

    outBuf = inBuf

    logging.info("-- Generating footnote landing zones (lzdestt={} lzdesth={})".format(lzdestt, lzdesth))

    if not lzdestt in ("chapterend", "bookend", ""):
        logging.error("Unrecognized value for --lzdestt ({})".format(lzdestt))
    if not lzdesth in ("chapterend", "bookend", ""):
        logging.error("Unrecognized value for --lzdesth ({})".format(lzdesth))

    if lzdestt == "bookend" or lzdesth == "bookend":
        lzs = ""
        fnMarkup = []

        if lzdestt == "bookend":
            lzs += "t"
        if lzdesth == "bookend":
            lzs += "h"

        if len(lzs) == 1:
            fnMarkup.append(".if {}".format(lzs))
        fnMarkup.append(".sp 4")
        fnMarkup.append(".pb")
        fnMarkup.append(".de div.footnotes { border: dashed 1px #aaaaaa; padding: 1.5em; }")
        fnMarkup.append(".de div.footnotes h2 { margin-top: 1em; }")
        fnMarkup.append('.dv class="footnotes"')
        fnMarkup.append(".sp 2")
        fnMarkup.append(".h2 id=footnotes nobreak")
        fnMarkup.append("FOOTNOTES:")
        fnMarkup.append(".sp 2")
        fnMarkup.append(".fm rend=no lz={}".format(lzs))
        fnMarkup.append(".dv-")
        if len(lzs) == 1:
            fnMarkup.append(".if-")

        outBuf.extend(fnMarkup)

    if lzdestt == "chapterend" or lzdesth == "chapterend":
        lzs = ""
        if lzdestt == "chapterend":
            lzs += "t"
        if lzdesth == "chapterend":
            lzs += "h"

        nextChapterStart = findNextChapter(outBuf, 0)
        while nextChapterStart:
            # Find end of chapter (line after last line of last paragraph)
            # Chapter headings must be marked in ppgen format (.h2)
            lastChapterEnd = findPreviousEmptyLine(outBuf, nextChapterStart)
            outBuf.insert(lastChapterEnd, ".fm lz={}".format(lzs))
            nextChapterStart = findNextChapter(outBuf, nextChapterStart+2)

    return outBuf


def joinSpannedFormatting(inBuf, keepOriginal):
    outBuf = []

    logging.info("Joining spanned out-of-line formatting markup")

    # Find:
    # 1: */
    # 2: // 010.png
    # 3:
    # 4: /*

    # Replace with:
    # 2: // 010.png
    # 3:

    lineNum = 0
    joinCount = 0
    while lineNum < len(inBuf):
        joinWasMade = False

        m = re.match(r"(\*\/|\#\/)$", inBuf[lineNum])
        if m:
            outBlock = []
            ln = lineNum + 1
            joinEndLineRegex = r"^\/\{}$".format(m.group(1)[0])
            while ln < len(inBuf) and isLineBlank(inBuf[ln]):
                outBlock.append(inBuf[ln])
                ln += 1

            if ln < len(inBuf) and isLinePageBreak(inBuf[ln]):
                outBlock.append(inBuf[ln])
                ln += 1
                while ln < len(inBuf)-1 and isLineBlank(inBuf[ln]) or re.match(r".pn", inBuf[ln]) or re.match(r"\/\/", inBuf[ln]):
                    outBlock.append(inBuf[ln])
                    ln += 1

                if re.match(joinEndLineRegex, inBuf[ln]) and (ln-1)-findPreviousNonEmptyLine(inBuf, ln-1) < 4:
                    for line in outBlock:
                        outBuf.append(line)
                    joinWasMade = True
                    joinCount += 1
                    logging.debug("Lines {}, {}: Joined spanned markup /{} {}/".format(lineNum+1, ln, m.group(1)[0], m.group(1)[0]))
                    lineNum = ln + 1

        if not joinWasMade:
            outBuf.append(inBuf[lineNum])
            lineNum += 1

    logging.info("Joined {} instances of spanned out-of-line formatting markup".format(joinCount))
    return outBuf


def buildImageDictionary():
    # Build dictionary of image files in images/ directory
    files = sorted(glob.glob("images/*"))

    logging.info("--- Taking inventory of /image folder")
    images = {}
    for f in files:
        try:
            img = Image.open(f)
            img.load()
        except IOError:
            logging.warning("Error loading '{}' ... skipping".format(f))
        except:
            raise
        else:
            fn = os.path.basename(f)
            anchorID = idFromFilename(fn)
            logging.debug("Found image id={} fn='{}' size={}".format(anchorID, fn, img.size))
            scanPageNum = re.sub("[^0-9]", "", fn)
            key = idFromFilename(fn)
            images[key] = ({'anchorID':anchorID, 'fileName':fn, 'scanPageNum':scanPageNum, 'dimensions':img.size, 'caption':"", 'usageCount':0 })

            if not re.match(r"i_\d{3,4}[a-z]?\.", fn) and fn != "cover.jpg":
                logging.warning("File '{}' does not match expected naming convention (i_001, i_001a)".format(fn))

#   print(images)
    logging.info("----- Found {} images".format(len(images)))

    return images


def idFromFilename(fn):
    id = os.path.basename(fn) # strip to filename only
    id = os.path.splitext(id)[0] # strip off extension
    return id

def idFromPageNumber(pn):
    s =  'i_{}'.format(pn)
    return s

def processIllustrations(inBuf):
    # Replace [Illustration: caption] markup with equivalent .il/.ca statements
    outBuf = []
    lineNum = 0
    currentScanPage = 0
    illustrationTagCount = 0
    asteriskIllustrationTagCount = 0
    #TODO use format() instead of +

    logging.info("-- Processing illustrations")

    illustrations = buildImageDictionary()

    logging.info("--- Converting [Illustration] tags")
    while lineNum < len(inBuf):
        # Keep track of active scanpage, page numbers must be
        pn = parseScanPage(inBuf[lineNum])
        if pn:
            currentScanPage = os.path.splitext(pn)[0]

        # Copy until next illustration block
        if re.match(r"\[Illustration", inBuf[lineNum]) or re.match(r"\*\[Illustration", inBuf[lineNum]):
            inBlock = []
            outBlock = []

            # *[Illustration:] tags need to be handled manually afterward (can't reposition before or illustration will change page location)
            ilNeedsRelocation = False
            if re.match(r"\*\[Illustration", inBuf[lineNum]):
                asteriskIllustrationTagCount += 1
                ilNeedsRelocation = True
            else:
                illustrationTagCount += 1

            # Copy illustration block
            bracketLevel = 0
            done = False
            while lineNum < len(inBuf)-1 and not done:
                # Detect when [ ] level so that nested [] are handled properly
                m = re.findall("\[", inBuf[lineNum])
                for b in m:
                    bracketLevel += 1
                m = re.findall("\]", inBuf[lineNum])
                for b in m:
                    bracketLevel -= 1

                if bracketLevel == 0 and re.search(r"]$", inBuf[lineNum]):
                    done = True
                    inBlock.append(inBuf[lineNum])
                    lineNum += 1
                else:
                    inBlock.append(inBuf[lineNum])
                    lineNum += 1

            # Handle multiple illustrations per page, must be named (i_001a, i_001b, ...) or (i_001, i_001a, i_001b, ...)
            ilID = None
            testID = idFromPageNumber(currentScanPage)
            if testID in illustrations and illustrations[testID]['usageCount'] == 0:
                ilID = testID
            else: # try i_001a, i_001b, ..., i_001z
                alphabet = map(chr, range(97, 123))
                for letter in alphabet:
                    if testID+letter in illustrations and illustrations[testID+letter]['usageCount'] == 0:
                        ilID = testID+letter
                        break

            if ilID is None and testID in illustrations:
                ilID = testID
            elif ilID is None:
                logging.error("No image file for illustration located on scan page {}".format(currentScanPage))

            if ilNeedsRelocation:
                outBlock.append("// *** DP2PPGEN *[Illustration] NEEDS RELOCATION ***")

            if ilID:
                # Convert to ppgen illustration block
                # .il id=i001 fn=i_001.jpg w=600 alt=''
                outBlock.append(".il id={} fn={} w={}px alt=''".format(ilID, illustrations[ilID]['fileName'], str(illustrations[ilID]['dimensions'][0])))
                illustrations[ilID]['usageCount'] += 1
            else:
                outBlock.append(".il id={} fn={}.jpg alt=''".format(testID, testID))

            # Extract caption from illustration block
            captionBlock = []
            for line in inBlock:
                if line == "]":
                    continue
                line = re.sub(r"^\*?\[Illustration: ?", "", line)
                line = re.sub(r"^\*?\[Illustration", "", line)
                line = re.sub(r"]$", "", line)
                captionBlock.append(line)

            # .ca SOUTHAMPTON BAR IN THE OLDEN TIME.
            if len(captionBlock) == 0 or (len(captionBlock) == 1 and captionBlock[0] == ""):
                # No caption
                pass
            elif len(captionBlock) == 1:
                # One line caption
                outBlock.append(".ca " + captionBlock[0])
            else:
                # Multiline caption
                outBlock.append(".ca")
                for line in captionBlock:
                    outBlock.append(line)
                outBlock.append(".ca-")

            # Write out ppgen illustration block
            for line in outBlock:
                outBuf.append(line)

            logging.debug("{}: ScanPage {}: convert {}".format(str(lineNum+1), str(currentScanPage), str(inBlock)))
        else:
            outBuf.append(inBuf[lineNum])
            lineNum += 1

    logging.info("--- Processed {} [Illustrations] tags".format(illustrationTagCount))
    if asteriskIllustrationTagCount > 0:
        logging.warning("Found {} *[Illustrations] tags; ppgen .il/.ca statements have been generated, but relocation to paragraph break must be performed manually.".format(asteriskIllustrationTagCount))

    return outBuf


def getLinesUntil(inBuf, startLineNum, endRegex, direction=1):
    outBlock = []
    lineNum = startLineNum

    while lineNum >= 0 and lineNum < len(inBuf) and not re.search(endRegex, inBuf[lineNum]):
        outBlock.append(inBuf[lineNum])
        lineNum += 1

    return outBlock


def joinSpannedHyphenations(inBuf, keepOriginal):
    outBuf = []

    logging.info("Joining spanned hyphenations")

    # Find:
    # 1: the last word on this line is cont-*
    # 2: // 010.png
    # 3: *inued. on the line below

    # 4: the last word on this line is emdash--*
    # 5: // 010.png
    # 6: This is the line below

    # 7: the first word on the next page is emdash
    # 8: // 010.png
    # 9: *--This is the line below

    # 10: the last word on this line is <i>cont-*</i>
    # 11: // 010.png
    # 12: <i>*-inued</i>. on the line below

    # Replace with:
    # 1: the last word on this line is cont-**inued.
    # 2: // 010.png
    # 3: on the line below

    # 4: the last word on this line is emdash--This
    # 5: // 010.png
    # 6: is the line below

    # 7: the first word on the next page is emdash--This
    # 8: // 010.png
    # 9: is the line below

    # 10: the last word on this line is <i>cont-**inued</i>.
    # 11: // 010.png
    # 12: on the line below

    lineNum = 0
    joinCount = 0
    nowrapLevel = 0
    while lineNum < len(inBuf):
        needsJoin = False
        joinToLineNum = 0
        joinFromLineNum = 0
        solInlineMarkup = ""
        eolInlineMarkup = ""

        if inBuf[lineNum].startswith("/*"):
            nowrapLevel += 1
        elif inBuf[lineNum].startswith("*/"):
            nowrapLevel -= 1

        # spanned hyphenation
        #TODO skip multiline [] markup between spanned hyphenation

        m = re.search(r"(?<![-—])-\*?(<\/(i|b|sc|g|f)>)?$", inBuf[lineNum])
        if m and lineNum < len(inBuf)-1 and isLinePageBreak(inBuf[lineNum+1]):
            eolInlineMarkup = m.group(1)
            if inBuf[lineNum][-1] == "*" or inBuf[lineNum].endswith("*{}".format(eolInlineMarkup)):
                #logging.debug("spanned hyphenation found: {}".format(inBuf[lineNum]))
                joinToLineNum = lineNum
                joinFromLineNum = findNextLineOfText(inBuf, lineNum+1)
                m = re.match(r"\*?(<(i|b|sc|g|f)>)", inBuf[joinFromLineNum])
                if m:
                    solInlineMarkup = m.group(1)
                if inBuf[joinFromLineNum][0] != '*' and not inBuf[joinFromLineNum].startswith("{}*".format(solInlineMarkup)):
                    logging.error("Line {}: Unresolved hyphenation\n       {}\n       {}".format(lineNum+1, inBuf[joinToLineNum], inBuf[joinFromLineNum]))
                else:
                    needsJoin = True
            elif not isDotCommand(inBuf[lineNum]) and not isLinePageBreak(inBuf[lineNum]):
                logging.warning("Line {}: Unmarked end of line hyphenation\n         {}".format(lineNum+1, inBuf[lineNum]))

        # em-dash / long dash end of last line
        if re.search(r"(?<![-—])(--|—)\*?$", inBuf[lineNum]) or re.search(r"(?<![-—])(----|——)\*?$", inBuf[lineNum]):
            if inBuf[lineNum][-1] == "*":
                #logging.debug("end of line emdash found: {}".format(inBuf[lineNum]))
                joinToLineNum = lineNum
                joinFromLineNum = findNextLineOfText(inBuf, lineNum+1)
                needsJoin = True
            elif nowrapLevel == 0 and not isNextOriginalLineBlank(inBuf, lineNum+1) and not isLinePageBreak(inBuf[lineNum]):
                logging.warning("Line {}: Unclothed end of line dashes\n         {}".format(lineNum+1, inBuf[lineNum]))

        # em-dash / long dash start of first line
        if re.match(r"\*?(--|—)(?![-—])", inBuf[lineNum]) or re.match(r"\*?(----|——)(?![-—])", inBuf[lineNum]):
            if inBuf[lineNum][0] == "*":
                #logging.debug("start of line emdash found: {}".format(inBuf[lineNum]))
                joinToLineNum = findPreviousLineOfText(inBuf, lineNum-1)
                joinFromLineNum = lineNum
                needsJoin = True
            elif nowrapLevel == 0 and not isPreviousOriginalLineBlank(inBuf, lineNum-1) and not isLinePageBreak(inBuf[lineNum]):
                logging.warning("Line {}: Unclothed start of line dashes\n         {}".format(lineNum+1, inBuf[lineNum]))

        if needsJoin:
            #logging.debug("  joinToLineNum: {}".format(inBuf[joinToLineNum]))
            #logging.debug("joinFromLineNum: {}".format(inBuf[joinFromLineNum]))
            # Remove first word of from line
            fromWord = inBuf[joinFromLineNum].split(' ', 1)[0]
            if len(inBuf[joinFromLineNum].split(' ', 1)) > 1:
                inBuf[joinFromLineNum] = inBuf[joinFromLineNum].split(' ', 1)[1]
            else:
                # Single word on from line, remove blank line
                del inBuf[joinFromLineNum]

            # Append it to toline
            if joinToLineNum == lineNum:
                inBuf[joinToLineNum] = inBuf[joinToLineNum] + fromWord
            else:
                dl = -1*(joinFromLineNum-joinToLineNum)
                outBuf[dl] = outBuf[dl] + fromWord

            # Collapse inline markup inside join
            if not eolInlineMarkup:
                eolInlineMarkup = solInlineMarkup.replace('<', '</')
            inBuf[joinToLineNum] = inBuf[joinToLineNum].replace('-*{}{}*'.format(eolInlineMarkup, solInlineMarkup), '-**')
            inBuf[joinToLineNum] = inBuf[joinToLineNum].replace('{}-**{}'.format(eolInlineMarkup, solInlineMarkup), '-**')
            inBuf[joinToLineNum] = inBuf[joinToLineNum].replace('-{}**{}'.format(eolInlineMarkup, solInlineMarkup), '-**')
            inBuf[joinToLineNum] = inBuf[joinToLineNum].replace('-*{}*{}'.format(eolInlineMarkup, solInlineMarkup), '-**')
            inBuf[joinToLineNum] = inBuf[joinToLineNum].replace('-*{}{}*'.format(eolInlineMarkup, solInlineMarkup), '-**')

            logging.debug("{}: Resolved hyphenation, ...{}".format(joinToLineNum+1, inBuf[joinToLineNum][-30:]))
            joinCount += 1

        outBuf.append(inBuf[lineNum])
        lineNum += 1

    logging.info("Joined {} instances of spanned hyphenations".format(joinCount))
    return outBuf


def addBoilerplate(inBuf):
    outBuf = inBuf

    headerBlock = []
    fn = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'header.txt')
    logging.info("Adding boilerplate from {}".format(fn))
    try:
        with open(fn, 'r') as f:
            for line in f:
                headerBlock.append(line.rstrip())
    except (IOError, OSError) as e:
        logging.info("Couldn't load {}, skipping header boilerplate".format(fn))

    outBuf[0:0] = headerBlock

    footerBlock = []
    fn = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'footer.txt')
    logging.info("Adding boilerplate from {}".format(fn))
    try:
        with open(fn, 'r') as f:
            for line in f:
                headerBlock.append(line.rstrip())
    except (IOError, OSError) as e:
        logging.info("Couldn't load {}, skipping footer boilerplate".format(fn))

    outBuf.extend(footerBlock)

    return outBuf


def tabsToSpaces(inBuf, tabSize):
    outBuf = []

    for line in inBuf:
        spaces = " " * tabSize
        line = line.replace("\t", spaces)
        outBuf.append(line)

    return outBuf


def convertUTF8(inBuf):
    outBuf = []
    lineCount = 0

    logging.info("Converting characters to UTF-8")

    for i, line in enumerate(inBuf):
        originalLine = line
        if not isLinePageBreak(line):
            # -- becomes a unicode mdash, ---- becomes 2 unicode mdashes
            line = re.sub(r"(?<!-)-{2}(?!-)", "—", line)
            line = re.sub(r"(?<!-)-{4}(?!-)", "——", line)
            if "--" in line:
                logging.warn("Unconverted dashes: {}".format(line))

        # [oe] becomes œ
        # [OE] becomes Œ
        line = line.replace("[oe]", "œ")
        line = line.replace("[OE]", "Œ")

        if line != originalLine:
            lineCount += 1
            logging.debug("{}: {}".format(i, originalLine))
            logging.debug("{}{}".format(" "*(len(str(i))+2), line))

        outBuf.append(line)

        # Fractions?

    logging.info("Converted characters on {} lines to UTF-8".format(lineCount))
    return outBuf


def convertThoughtBreaks(inBuf):
    outBuf = []

    for line in inBuf:
        # <tb> to .tb
        line = re.sub(r"^<tb>$", ".tb", line)
        outBuf.append(line)

    return outBuf


def removeBlankLinesAtPageEnds(inBuf):
    outBuf = []

    for line in inBuf:
        if isLinePageBreak(line):
            while outBuf and isLineBlank(outBuf[-1]):
                outBuf.pop()

        outBuf.append(line)

    return outBuf


# TODO: Make this a tool in itself?
def fixup(inBuf, keepOriginal):
#    • Remove spaces at end of line.
#    • Remove blank lines at end of pages.
#    • Remove spaces on either side of hyphens.
#    • Remove space before periods.
#    • Remove space before exclamation points.
#    • Remove space before question marks.
#    • Remove space before commas.
#    • Remove space before semicolons.
#    • Remove space before colons.
#    • Remove space after opening and before closing brackets. () [] {}
#    • Remove space after open angle quote and before close angle quote.
#    • Remove space after beginning and before ending double quote.
#    • Ensure space before ellipses except after period.
#    • Format any line that contains only 5 *s and whitespace to be the standard 5 asterisk thought break.
#    • Convert multiple space to single space.
#    • Fix obvious l<-->1 problems.
#    You can also specify whether to skip text inside the /* */ markers or not.


    outBuf = inBuf

    outBuf = tabsToSpaces(outBuf, 4)
    outBuf = removeTrailingSpaces(outBuf)
    outBuf = convertThoughtBreaks(outBuf)
    outBuf = removeBlankLinesAtPageEnds(outBuf)
#   outBuf = removeExtraSpaces(outBuf)

    return outBuf

#TODO: Full guiguts fixit seems error prone.. maybe only do safe defaults or break off into seperate tool with each setting configurable, does gutsweeper do this already?
#def removeExtraSpaces(inBuf):
#    • Remove spaces on either side of hyphens.
#    • Remove space before periods.
#    • Remove space before exclamation points.
#    • Remove space before question marks.
#    • Remove space before commas.
#    • Remove space before semicolons.
#    • Remove space before colons.
#    • Remove space after opening and before closing brackets. () [] {}
#    • Remove space after open angle quote and before close angle quote.
#    • Remove space after beginning and before ending double quote.
#    • Ensure space before ellipses except after period.
#   rewrapLevel = 0
#   for line in inBuf:
#       # Detect when inside out-of-line formatting block /# #/ /* */
#       if re.match(r"\/[\*\#]", inBuf[lineNum]):
#           rewrapLevel += 1
#       elif re.match(r"[\*\#]\/", inBuf[lineNum]):
#           rewrapLevel -= 1
#
#       if rewrapLevel == 0:
#           # Remove multiple spaces
#           # $line =~ s/(?<=\S)\s\s+(?=\S)/
#           line = re.sub(r"(?<=\S)\s\s+(?=\S)", "", line)
#
#           # Remove spaces on either side of hyphens.
#           # Remove spaces before hyphen (only if hyphen isn't first on line, like poetry)
#           # $line =~ s/(\S) +-/$1-/g;
#           line = re.sub(r"(\S) +-", "\1-", line)
#
#           # Remove space after hyphen
#           # $line =~ s/- /-/g;
#           line = re.sub(r"- ", "-", line)
#
#           # Except leave a space after a string of three or more hyphens
#           # $line =~ s/(?<![-])([-]*---)(?=[^\s\\"F-])/$1 /g
#           line = re.sub(r'(?<!-)(-*---)(?=[^\s\\"F-])', "\1", line)
#
#       outBuf.append(line)
#
#   return outBuf
#
#               $edited++ if $line =~ s/- /-/g;    # Remove space after hyphen
#               $edited++
#                 if $line =~ s/(?<![-])([-]*---)(?=[^\s\\"F-])/$1 /g
#               ; # Except leave a space after a string of three or more hyphens
#
#
#
#           if (${ $::lglobal{fixopt} }[1]) {
#               ; # Remove spaces before hyphen (only if hyphen isn't first on line, like poetry)
#               $edited++ if $line =~ s/(\S) +-/$1-/g;
#               $edited++ if $line =~ s/- /-/g;    # Remove space after hyphen
#               $edited++
#                 if $line =~ s/(?<![-])([-]*---)(?=[^\s\\"F-])/$1 /g
#               ; # Except leave a space after a string of three or more hyphens
#           }
#           if (${ $::lglobal{fixopt} }[3]) {
#               ; # Remove space before periods (only if not first on line, like poetry's ellipses)
#               $edited++ if $line =~ s/(\S) +\.(?=\D)/$1\./g;
#           }
#           ;     # Get rid of space before periods
#           if (${ $::lglobal{fixopt} }[4]) {
#               $edited++
#                 if $line =~ s/ +!/!/g;
#           }
#           ;     # Get rid of space before exclamation points
#           if (${ $::lglobal{fixopt} }[5]) {
#               $edited++
#                 if $line =~ s/ +\?/\?/g;
#           }
#           ;     # Get rid of space before question marks
#           if (${ $::lglobal{fixopt} }[6]) {
#               $edited++
#                 if $line =~ s/ +\;/\;/g;
#           }
#           ;     # Get rid of space before semicolons
#           if (${ $::lglobal{fixopt} }[7]) {
#               $edited++
#                 if $line =~ s/ +:/:/g;
#           }
#           ;     # Get rid of space before colons
#           if (${ $::lglobal{fixopt} }[8]) {
#               $edited++
#                 if $line =~ s/ +, /, /g;
#           }
#           ;     # Get rid of space before commas
#                 # FIXME way to go on managing quotes
#           if (${ $::lglobal{fixopt} }[9]) {
#               $edited++
#                 if $line =~ s/^\" +/\"/
#               ; # Remove space after doublequote if it is the first character on a line
#               $edited++
#                 if $line =~ s/ +\"$/\"/
#               ; # Remove space before doublequote if it is the last character on a line
#           }
#           if (${ $::lglobal{fixopt} }[10]) {
#               $edited++
#                 if $line =~ s/(?<=(\(|\{|\[)) //g
#               ;    # Get rid of space after opening brackets
#               $edited++
#                 if $line =~ s/ (?=(\)|\}|\]))//g
#               ;    # Get rid of space before closing brackets
#           }
#           ;        # FIXME format to standard thought breaks - changed to <tb>
#           if (${ $::lglobal{fixopt} }[11]) {
#               $edited++
#
#          #                  if $line =~
#          # s/^\s*(\*\s*){5}$/       \*       \*       \*       \*       \*\n/;
#                 if $line =~ s/^\s*(\*\s*){4,}$/<tb>\n/;
#           }
#           $edited++ if ($line =~ s/ +$//);
#           ;        # Fix llth, lst
#           if (${ $::lglobal{fixopt} }[12]) {
#               $edited++ if $line =~ s/llth/11th/g;
#               $edited++ if $line =~ s/(?<=\d)lst/1st/g;
#               $edited++ if $line =~ s/(?<=\s)lst/1st/g;
#               $edited++ if $line =~ s/^lst/1st/;
#           }
#           ;        # format ellipses correctly
#           if (${ $::lglobal{fixopt} }[13]) {
#               $edited++ if $line =~ s/(?<![\.\!\?])\.{3}(?!\.)/ \.\.\./g;
#               $edited++ if $line =~ s/^ \./\./;
#           }
#           ;        # format guillemets correctly
#           ;        # french guillemets
#           if (${ $::lglobal{fixopt} }[14] and ${ $::lglobal{fixopt} }[15]) {
#               $edited++ if $line =~ s/«\s+/«/g;
#               $edited++ if $line =~ s/\s+»/»/g;
#           }
#           ;        # german guillemets
#           if (${ $::lglobal{fixopt} }[14] and !${ $::lglobal{fixopt} }[15])
#           {
#               $edited++ if $line =~ s/\s+«/«/g;
#               $edited++ if $line =~ s/»\s+/»/g;
#           }
#           $update++ if (($index % 250) == 0);
#           $textwindow->see($index) if ($edited || $update);
#           if ($edited) {
#               $textwindow->replacewith($lastindex, $index, $line);
#           }
#       }
#       $textwindow->markSet('insert', $index) if $update;
#       $textwindow->update   if ($edited || $update);
#       ::update_indicators() if ($edited || $update);
#       $edited    = 0;
#       $update    = 0;
#       $lastindex = $index;
#       $index++;
#       $index .= '.0';
#       if ($index > $end) { $index = $end }
#       if ($::operationinterrupt) { $::operationinterrupt = 0; return }
#   }
#   $textwindow->markSet('insert', 'end');
#   $textwindow->see('end');
#   ::update_indicators();
#}

def doStandardConversions(inBuf, keepOriginal):
    outBuf = inBuf

    outBuf = removeTrailingSpaces(outBuf)
    outBuf = convertThoughtBreaks(outBuf)

    return outBuf


def generateTransNote(inBuf):
    outBuf = []
    tnote = []

    pageNumbers = calcPageNumbers(inBuf)
#   print(pageNumbers)
    lineNum = 0
    currentPageNum = 0
    currentScanPage = 0
    isPageNumRoman = True

    logging.info("-- Generating transcriber's note")
    while lineNum < len(inBuf):
        # Keep track of active scanpage
        if isLinePageBreak(inBuf[lineNum]):
            currentScanPage = parseScanPage(inBuf[lineNum])

        # Look for proofers notes [**
        m = re.search(r"\[\*\*([^\]]+)]", inBuf[lineNum])
        if m:
            #print("[{}] {}".format(pageNumbers[currentScanPage]['pageNum'], inBuf[lineNum]))
            logging.info("[**note] found {}: {}".format(lineNum,m.group(0)))

            if not pageNumbers:
                logging.warning("Page number for proofers note could not be determined, add .pn statements and run again")
                pageLabel = "???"
            elif pageNumbers[currentScanPage]['isPageNumRoman']:
                pageLabel = str(toRoman(pageNumbers[currentScanPage]['pageNum']))
            else:
                pageLabel = str(pageNumbers[currentScanPage]['pageNum'])

            t = m.group(1).split('|')
            if len(t) > 2: # sanity check
                fatal("Error parsing proofer note [{}] {}".format(pageNumbers[currentScanPage]['pageNum'], m.group(0)))

            beforeText = "{}".format(inBuf[lineNum][m.start(0)-20:m.start(0)])
            afterText = "{}".format(inBuf[lineNum][m.end(0):m.end(0)+20])

            try:
                beforeText = beforeText.split(' ', 1)[1]
            except IndexError as e:
                beforeText = "{}".format(inBuf[lineNum][:m.start(0)])

            try:
                afterText = afterText.rsplit(' ', 1)[0]
            except IndexError as e:
                afterText = "{}".format(inBuf[lineNum][m.end(0):])

            #print(inBuf[lineNum])
            if len(t) == 1: # Non-substitution type note, or unprocessed note
                tnote.append("• #{}:Page_{}#".format(m.group(0), pageLabel))
            else:
                if t[0] == "":
                    oldText = "{}{}{}".format(beforeText, t[0], afterText)
                else:
                    oldText = "{}<B>{}</B>{}".format(beforeText, t[0], afterText)

                if t[1] == "":
                    newText = "{}{}{}".format(beforeText, t[1], afterText)
                else:
                    newText = "{}<B>{}</B>{}".format(beforeText, t[1], afterText)

                newText = "{}<B>{}</B>{}".format(beforeText, t[1], afterText)
                tnote.append("#Page {}:tnote_{}#: {} → {}".format(pageLabel, lineNum, oldText, newText))

                inBuf[lineNum] = inBuf[lineNum].replace(m.group(0), "<span id='tnote_{}'>{}</span>".format(lineNum, t[1]))

        outBuf.append(inBuf[lineNum])
        lineNum += 1

    outBuf.append("")
    outBuf.append("// Transcriber’s note")
    outBuf.append(".sp 4")
    outBuf.append(".pb")
    outBuf.append(".na")
    outBuf.append(".ni")
    outBuf.append(".de div.transnote { background-color: #eef; border: dashed 1px #aaa; color: black; padding: 1em; font-family: sans-serif, serif; }")
    outBuf.append(".de div.transnote h2 { margin-top: 1em; }")
    outBuf.append(".ma • *")
    outBuf.append('.ma → "->"')
    outBuf.append('.dv class="transnote"')
    outBuf.append(".sp 2")
    outBuf.append(".h2 nobreak //id=transnote")
    outBuf.append("TRANSCRIBER’S NOTE:")
    outBuf.append(".sp 2")
    outBuf.append(".in 1")
    outBuf.append("• Silently corrected obvious punctuation and capitalization errors.")
    outBuf.append("")
    outBuf.append("• Other changes:")
    outBuf.append(".in +2")
    outBuf.append(".nf l")

    outBuf.extend(tnote)

    outBuf.append(".nf-")
    outBuf.append(".in")
    outBuf.append(".in")
    outBuf.append(".dv-")
    outBuf.append("")

    return outBuf


def stripHtml(inBuf):
    outBuf = []
    for line in inBuf:
        line = re.sub("</?[^>]+>", "", line)

        outBuf.append(line)

    return outBuf


# -------------------------------------------------------------------------------------
# Roman numeral processsing

# Define digit mapping
romanNumeralMap = (('m',  1000), ('cm', 900), ('d',  500), ('cd', 400), ('c',  100),
    ('xc', 90), ('l',  50), ('xl', 40), ('x',  10), ('ix', 9), ('v',  5),
    ('iv', 4), ('i',  1))

def toRoman(n):
    """convert integer to Roman numeral"""
    result = ""
    for numeral, integer in romanNumeralMap:
        while n >= integer:
            result += numeral
            n -= integer
    return result

def fromRoman(s):
    """convert Roman numeral to integer"""
    result = 0
    index = 0
    for numeral, integer in romanNumeralMap:
        while s[index:index+len(numeral)] == numeral:
            result += integer
            index += len(numeral)
    return result


def calcPageNumbers(inBuf):
    pageNumbers = {}
    lineNum = 0
    currentPageNum = 0
    currentScanPage = 0
    isPageNumRoman = True

    logging.info("-- Calculating page numbers")
    while lineNum < len(inBuf):
        # Keep track of page number
        m = re.match(r".pn (.+)", inBuf[lineNum])
        if m:
            pn = "{}".format(m.group(1))
            if pn[0] == '+':
                d = int(pn[1:])
                currentPageNum += d

            elif pn[0] in [i[0][0] for i in romanNumeralMap]:
                num = fromRoman(pn)
                currentPageNum = int(num)
                isPageNumRoman = True
            else:
                currentPageNum = int(pn)
                isPageNumRoman = False

#           print("[{}] {} {} {}".format(currentScanPage, currentPageNum, d, isPageNumRoman))
            pageNumbers[currentScanPage] = ({'pageNum':currentPageNum, 'isPageNumRoman':isPageNumRoman })

        # Keep track of active scanpage
        if isLinePageBreak(inBuf[lineNum]):
            currentScanPage = parseScanPage(inBuf[lineNum])

        lineNum += 1

    return pageNumbers

def autoFixHyphens(inBuf):
    logging.info("-- Automatically fixing hyphenation issues")
    hyphenIssues = analyzeHyphenation(inBuf)

    for h in hyphenIssues:
        s = '{}{}{}'.format(h['firstWord'],h['hyphens'],h['secondWord'])
        r = None
        if h['hyphens'] == '-*':
            if h['usageWithHyphen'] == 0 and h['usageWithoutHyphen'] > 0:
                r = '{}{}'.format(h['firstWord'],h['secondWord'])
            elif h['usageWithoutHyphen'] == 0 and h['usageWithHyphen'] > 0:
                r = '{}-{}'.format(h['firstWord'],h['secondWord'])

        elif h['hyphens'] == '-**':
            if h['usageWithHyphen'] == 0 and h['usageWithoutHyphen'] > 0:
                r = '{}{}'.format(h['firstWord'],h['secondWord'])
            elif h['usageWithoutHyphen'] == 0 and h['usageWithHyphen'] > 0:
                r = '{}-{}'.format(h['firstWord'],h['secondWord'])

        if r is not None:
            inBuf[h['lineNum']] = inBuf[h['lineNum']].replace(s,r)
            logging.info("[{}] Replaced {} with {} ({} vs {})".format(h['lineNum'],s,r,h['usageWithHyphen'],h['usageWithoutHyphen']))
        else:
            logging.info("[{}] Skipped {} ({} vs {})".format(h['lineNum'],s,h['usageWithHyphen'],h['usageWithoutHyphen']))

    return inBuf


def analyzeHyphenation(inBuf):
    logging.info("-- Analyzing hyphenation")
    logging.warning("Multiple matches on the same line will only be counted as one")

    hyphenation = []
    lineNum = 0

    while lineNum < len(inBuf):
        m = re.findall(r"(\w+)(\*+-|-\*+)(\w+)", inBuf[lineNum])
        for match in m:
            firstWord = match[0]
            hyphens = match[1]
            secondWord = match[2]

            unhyphenatedWord = '{}{}'.format(firstWord,secondWord).lower()
            hyphenatedWord = '{}-{}'.format(firstWord,secondWord).lower()

            # Much slower version (~10x), but properly detects multiple uses on the same line
            #usageWithoutHyphen = sum([1 for line in inBuf for m in re.finditer(unhyphenatedWord,line)])
            #usageWithHyphen = sum([1 for line in inBuf for m in re.finditer(hyphenatedWord,line)])

            # Slower version (~5x) multiple matches on the same line will correctly be counted as multiple match
            #usageWithoutHyphen = sum([1 for line in inBuf for w in line.lower().split() if unhyphenatedWord in w])
            #usageWithHyphen = sum([1 for line in inBuf for w in line.lower().split() if hyphenatedWord in w])

            # Fast, but will only report one match maximum for each line of input (multiple matches on the same line will be counted as one match)
            usageWithoutHyphen = sum([1 for line in inBuf if unhyphenatedWord in line.lower()])
            usageWithHyphen = sum([1 for line in inBuf if hyphenatedWord in line.lower()])

            commonUsage = '??'
            if usageWithHyphen > usageWithoutHyphen:
                commonUsage = '{}-{}'.format(firstWord,secondWord)
            elif usageWithHyphen < usageWithoutHyphen:
                commonUsage = '{}{}'.format(firstWord,secondWord)

            hyphenation.append({'firstWord':firstWord,'hyphens':hyphens,'secondWord':secondWord,'lineNum':lineNum,'usageWithoutHyphen':usageWithoutHyphen,'usageWithHyphen':usageWithHyphen,'commonUsage':commonUsage})
        lineNum += 1

    return hyphenation


def generateReport(inBuf,reportFormat):

    def parseOutline(inBuf):
        outline = []
        lineNum = 0
        while lineNum < len(inBuf):
            m = re.match(r".h([0-9]) (.+)", inBuf[lineNum])
            if m:
                level = m.group(1)
                options = m.group(2)
                text = inBuf[lineNum+1]
                outline.append({'level':level,'options':options,'text':text,'lineNum':lineNum})
            lineNum += 1

        return outline

    def parseIllos(inBuf):
        illos = []
        lineNum = 0
        while lineNum < len(inBuf):
            m = re.match(r".il (.+)", inBuf[lineNum])
            if m:
                level = m.group(1)
                options = m.group(2)
                text = inBuf[lineNum+1]
                outline.append({'lineNum':lineNum,'scanPageNum':scanPageNum,'pageNum':pageNum,'fileName':fileName,'dimensions':dimensions})
            lineNum += 1

        return illos

    report = {}

    report['outline'] = parseOutline(inBuf)
    report['hyphenation'] = analyzeHyphenation(inBuf)

    w1 = max([len(str(r['lineNum'])) for r in report['hyphenation']])
    w2 = max([len(r['firstWord'])+len(r['hyphens'])+len(r['secondWord']) for r in report['hyphenation']])
    w3 = max(5,len('w/ -'))
    w4 = max(5,len('wo/ -'))
    w5 = max(w2,len('Common Case'))
    tableHeading = '{:<{}}  {:<{}}  {:<{}} {:<{}} {:<{}}'.format('Line',w1,'Word',w2,'w/-',w3,'wo/-',w4,'Common Case',w5)

    print('\n{:-<{}}'.format('---[Words marked with -*]-',len(tableHeading)))
    print(tableHeading)
    print('{:-<{}}'.format('',len(tableHeading)))
    for r in report['hyphenation']:
        if r['hyphens'] == '-*':
            print('{:<{}}  {:<{}}  {:<{}} {:<{}} {}'.format(r['lineNum'],w1,r['firstWord']+r['hyphens']+r['secondWord'],w2,r['usageWithHyphen'],w3,r['usageWithoutHyphen'],w4,r['commonUsage'],w5))

    print('\n{:-<{}}'.format('---[Words marked with -**]-',len(tableHeading)))
    print(tableHeading)
    print('{:-<{}}'.format('',len(tableHeading)))
    for r in report['hyphenation']:
        if r['hyphens'] == '-**':
            print('{:<{}}  {:<{}}  {:<{}} {:<{}} {}'.format(r['lineNum'],w1,r['firstWord']+r['hyphens']+r['secondWord'],w2,r['usageWithHyphen'],w3,r['usageWithoutHyphen'],w4,r['commonUsage'],w5))

    for r in report['outline']:
        print('yes')
        print('{}[[h{}{}] {}'.format(r['level'],r['level']*3,r['options'],r['text']))

#   toc outline
#   table of footnotes
#   table of illustrations (in ppgimg too?)
#   list of proofer notes
#   markup errors with associated line numbers
#   list each usage of all <i>, <cite>, <i><lang>, ...
#   list each usage of all <b>, <sc>, ...



    #print(report)



def main():
    args = docopt(__doc__, version="dp2ppgen v{}".format(__version__))

    # Process required command line arguments
    outfile = createOutputFileName(args['<infile>'])
    if args['<outfile>']:
        outfile = args['<outfile>']

    infile = args['<infile>']

    # Open source file and represent as an array of lines
    inBuf = loadFile(infile)

    # Configure logging
    logLevel = logging.INFO #default
    if args['--verbose']:
        logLevel = logging.DEBUG
    elif args['--quiet']:
        logLevel = logging.ERROR

    logging.basicConfig(format='%(levelname)s: %(message)s', level=logLevel)
    logging.debug(args)

    # Process processing options

    #TODO, load config file and use those options if one is present
    if args['--config']:
        args = mergeDict(args,loadJson(args['--config']))

    # Use default options if no processing options are set
    elif not args['--chapters'] and \
        not args['--sections'] and \
        not args['--footnotes'] and \
        not args['--sidenotes'] and \
        not args['--illustrations'] and \
        not args['--markup'] and \
        not args['--pages'] and \
        not args['--fixup'] and \
        not args['--utf8'] and \
        not args['--boilerplate'] and \
        not args['--report'] and \
        not args['--tnote'] and \
        not args['--autofixhyphens'] and \
        not args['--joinspanned']:

        logging.info("No processing options were given, using default options from defaults.json\n      Run 'dp2ppgen -h' for a full list of options")

        defaultConfig = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'defaults.json')
        args = mergeDict(args,loadJson(defaultConfig))

    # Process source document
    logging.info("Processing '{}'".format(infile))
    outBuf = inBuf

    if not args['--report']:
        errorCount = validateDpMarkup(inBuf)
        if errorCount > 0 and not args['--force']:
            fatal("Correct markup issues then re-run operation, or use --force to ignore markup errors")


    outBuf = doStandardConversions(outBuf, args['--keeporiginal'])

    if args['--pages']:
        outBuf = processBlankPages(outBuf, args['--keeporiginal'])
        outBuf = processPageNumbers(outBuf, args['--keeporiginal'])
    if args['--fixup']:
        outBuf = fixup(outBuf, args['--keeporiginal'])
    if args['--utf8']:
        outBuf = convertUTF8(outBuf)
    if args['--chapters'] or args['--sections']:
        outBuf = processHeadings(outBuf, args['--chapters'], args['--sections'], args['--keeporiginal'], args['--chaptermaxlines'], args['--sectionmaxlines'])
    if args['--sidenotes']:
        outBuf = processSidenotes(outBuf, args['--keeporiginal'], args['--snkeepbreaks'])
    if args['--illustrations']:
        outBuf = processIllustrations(outBuf)
    if args['--footnotes']:
        # Set defaults
        fndest = ""
        lzdestt = ""
        lzdesth = ""
        if args['--fndest']:
            fndest = args['--fndest']
        else:
            fndest = "paragraphend"
            lzdestt = "chapterend"
            lzdesth = "bookend"

        if args['--lzdestt']:
            lzdestt = args['--lzdestt']
        if args['--lzdesth']:
            lzdesth = args['--lzdesth']
        fnautonum = False
        if args['--fnautonum']:
            fnautonum = True

        outBuf = processFootnotes(outBuf, fndest, args['--keeporiginal'], lzdestt, lzdesth, fnautonum)
    if args['--joinspanned']:
        outBuf = joinSpannedFormatting(outBuf, args['--keeporiginal'])
        outBuf = joinSpannedHyphenations(outBuf, args['--keeporiginal'])
    if args['--autofixhyphens']:
        autoFixHyphens(outBuf)
    if args['--detectmarkup']:
        outBuf = detectMarkup(outBuf)
    if args['--markup']:
        outBuf = processOOLFMarkup(outBuf, args['--keeporiginal'])

    if args['--boilerplate']:
        outBuf = addBoilerplate(outBuf)

    if args['--tnote']:
        outBuf = generateTransNote(outBuf)

    if args['--report']:
        generateReport(outBuf,args['--report'])

    if not args['--dryrun']:
        logging.info("Saving output to '{}'".format(outfile))
        # Save file
        f = open(outfile, 'w')
        f.write('\n'.join(outBuf))
        f.close()

    return


def loadJson(fn):
    with open(fn) as f:
        data = json.load(f)

    return data


def mergeDict(dict_1, dict_2):
    """Merge two dictionaries.
    Values that evaluate to true take priority over falsy values.
    `dict_1` takes priority over `dict_2`.
    """
    return dict((str(key), dict_1.get(key) or dict_2.get(key))
                for key in set(dict_2) | set(dict_1))


if __name__ == "__main__":
    main()
