# Collate training data

# So, we have genre maps generated by the pagetagger, as well as
# page-level feature/token counts generated by the new workflow in
# PythonScripts/workflow.

# The function of this script is to collate the two and copy them
# over to pagedata, making sure that we only include files where we
# have matches on both sides. Also, in the process, I need to trim
# the last two pages from certain mdp files that mistakenly included
# pages for "notes" and "pagedata" that were really metadata.

import os

tokendir = "/Users/tunder/Dropbox/PythonScripts/workflow/trainslice"

mapdir = "/Users/tunder/Dropbox/pagedata/detailedmaps"

outmapdir = "/Users/tunder/Dropbox/pagedata/pagemaps"
outtokendir = "/Users/tunder/Dropbox/pagedata/pagefeatures"

tokenfiles = os.listdir(tokendir)
tokendict = dict()

for filename in tokenfiles:
	if filename.endswith("pg.tsv"):
		inpath = tokendir + "/" + filename
		with open(inpath, encoding = 'utf-8') as f:
			filelines = f.readlines()
		tokendict[filename] = filelines

mapfiles = os.listdir(mapdir)
mapdict = dict()

for filename in mapfiles:
	if not filename.startswith("."):
		inpath = mapdir + "/" + filename
		with open(inpath, encoding = 'utf-8') as f:
			filelines = f.readlines()
		mapdict[filename] = filelines

for filename, tokenlines in tokendict.items():
	strippedname = filename[:-11]
	if strippedname not in mapdict:
		print(strippedname + " is missing.")
		continue

	maplines = mapdict[strippedname]

	striplasttwo = False

	if filename.startswith("mdp"):
		trailers = 0
		for line in maplines[-2::1]:
			line = line.rstrip()
			fields = line.split('\t')
			if fields[-1] == "errat" or fields[-1] == "libra":
				trailers += 1

		lastpage = 0
		for line in reversed(tokenlines):
			line = line.rstrip()
			fields = line.split('\t')
			if int(fields[0]) > lastpage:
				lastpage = int(fields[0])
			if int(fields[0]) + 2 < lastpage:
				break
			else:
				if fields[1] == "percentage" or fields[1] == "implicit":
					trailers += 1

		if trailers > 3:
			striplasttwo = True

	lastpage = 0
	for line in maplines[1:]:
		fields = line.split('\t')
		if int(fields[0]) > lastpage:
			lastpage = int(fields[0])

	otherlastpage = 0
	for line in tokenlines:
		fields = line.split('\t')
		if int(fields[0]) > otherlastpage:
			otherlastpage = int(fields[0])

	if lastpage != otherlastpage:
		print(strippedname + " " + str(lastpage) + " " + str(otherlastpage))

	if striplasttwo:
		lastpage = lastpage - 2
		print(filename)

	outdatapath = outtokendir + "/" + strippedname + ".pg.tsv"
	with open(outdatapath, mode = "w", encoding = "utf-8") as f:
		for line in tokenlines:
			fields = line.split('\t')
			try:
				pagenumber = int(fields[0])
			except:
				pagenumber = 0

			if pagenumber <= lastpage:
				f.write(line)

	outmappath = outmapdir + "/" + strippedname + ".map"
	with open(outmappath, mode = "w", encoding = "utf-8") as f:
		for line in maplines:
			fields = line.split('\t')
			try:
				pagenumber = int(fields[0])
			except:
				pagenumber = 0

			if pagenumber <= lastpage:
				f.write(line)
















469bf6c5ca0dd5d8e6792a3f21e6fc29c4d98f07
<mask token>

<mask token>
for filename in tokenfiles:
    if filename.endswith('pg.tsv'):
        inpath = tokendir + '/' + filename
        with open(inpath, encoding='utf-8') as f:
            filelines = f.readlines()
        tokendict[filename] = filelines
<mask token>
for filename in mapfiles:
    if not filename.startswith('.'):
        inpath = mapdir + '/' + filename
        with open(inpath, encoding='utf-8') as f:
            filelines = f.readlines()
        mapdict[filename] = filelines
for filename, tokenlines in tokendict.items():
    strippedname = filename[:-11]
    if strippedname not in mapdict:
        print(strippedname + ' is missing.')
        continue
    maplines = mapdict[strippedname]
    striplasttwo = False
    if filename.startswith('mdp'):
        trailers = 0
        for line in maplines[-2::1]:
            line = line.rstrip()
            fields = line.split('\t')
            if fields[-1] == 'errat' or fields[-1] == 'libra':
                trailers += 1
        lastpage = 0
        for line in reversed(tokenlines):
            line = line.rstrip()
            fields = line.split('\t')
            if int(fields[0]) > lastpage:
                lastpage = int(fields[0])
            if int(fields[0]) + 2 < lastpage:
                break
            elif fields[1] == 'percentage' or fields[1] == 'implicit':
                trailers += 1
        if trailers > 3:
            striplasttwo = True
    lastpage = 0
    for line in maplines[1:]:
        fields = line.split('\t')
        if int(fields[0]) > lastpage:
            lastpage = int(fields[0])
    otherlastpage = 0
    for line in tokenlines:
        fields = line.split('\t')
        if int(fields[0]) > otherlastpage:
            otherlastpage = int(fields[0])
    if lastpage != otherlastpage:
        print(strippedname + ' ' + str(lastpage) + ' ' + str(otherlastpage))
    if striplasttwo:
        lastpage = lastpage - 2
        print(filename)
    outdatapath = outtokendir + '/' + strippedname + '.pg.tsv'
    with open(outdatapath, mode='w', encoding='utf-8') as f:
        for line in tokenlines:
            fields = line.split('\t')
            try:
                pagenumber = int(fields[0])
            except:
                pagenumber = 0
            if pagenumber <= lastpage:
                f.write(line)
    outmappath = outmapdir + '/' + strippedname + '.map'
    with open(outmappath, mode='w', encoding='utf-8') as f:
        for line in maplines:
            fields = line.split('\t')
            try:
                pagenumber = int(fields[0])
            except:
                pagenumber = 0
            if pagenumber <= lastpage:
                f.write(line)

<mask token>
tokendir = '/Users/tunder/Dropbox/PythonScripts/workflow/trainslice'
mapdir = '/Users/tunder/Dropbox/pagedata/detailedmaps'
outmapdir = '/Users/tunder/Dropbox/pagedata/pagemaps'
outtokendir = '/Users/tunder/Dropbox/pagedata/pagefeatures'
tokenfiles = os.listdir(tokendir)
tokendict = dict()
for filename in tokenfiles:
    if filename.endswith('pg.tsv'):
        inpath = tokendir + '/' + filename
        with open(inpath, encoding='utf-8') as f:
            filelines = f.readlines()
        tokendict[filename] = filelines
mapfiles = os.listdir(mapdir)
mapdict = dict()
for filename in mapfiles:
    if not filename.startswith('.'):
        inpath = mapdir + '/' + filename
        with open(inpath, encoding='utf-8') as f:
            filelines = f.readlines()
        mapdict[filename] = filelines
for filename, tokenlines in tokendict.items():
    strippedname = filename[:-11]
    if strippedname not in mapdict:
        print(strippedname + ' is missing.')
        continue
    maplines = mapdict[strippedname]
    striplasttwo = False
    if filename.startswith('mdp'):
        trailers = 0
        for line in maplines[-2::1]:
            line = line.rstrip()
            fields = line.split('\t')
            if fields[-1] == 'errat' or fields[-1] == 'libra':
                trailers += 1
        lastpage = 0
        for line in reversed(tokenlines):
            line = line.rstrip()
            fields = line.split('\t')
            if int(fields[0]) > lastpage:
                lastpage = int(fields[0])
            if int(fields[0]) + 2 < lastpage:
                break
            elif fields[1] == 'percentage' or fields[1] == 'implicit':
                trailers += 1
        if trailers > 3:
            striplasttwo = True
    lastpage = 0
    for line in maplines[1:]:
        fields = line.split('\t')
        if int(fields[0]) > lastpage:
            lastpage = int(fields[0])
    otherlastpage = 0
    for line in tokenlines:
        fields = line.split('\t')
        if int(fields[0]) > otherlastpage:
            otherlastpage = int(fields[0])
    if lastpage != otherlastpage:
        print(strippedname + ' ' + str(lastpage) + ' ' + str(otherlastpage))
    if striplasttwo:
        lastpage = lastpage - 2
        print(filename)
    outdatapath = outtokendir + '/' + strippedname + '.pg.tsv'
    with open(outdatapath, mode='w', encoding='utf-8') as f:
        for line in tokenlines:
            fields = line.split('\t')
            try:
                pagenumber = int(fields[0])
            except:
                pagenumber = 0
            if pagenumber <= lastpage:
                f.write(line)
    outmappath = outmapdir + '/' + strippedname + '.map'
    with open(outmappath, mode='w', encoding='utf-8') as f:
        for line in maplines:
            fields = line.split('\t')
            try:
                pagenumber = int(fields[0])
            except:
                pagenumber = 0
            if pagenumber <= lastpage:
                f.write(line)

import os
tokendir = '/Users/tunder/Dropbox/PythonScripts/workflow/trainslice'
mapdir = '/Users/tunder/Dropbox/pagedata/detailedmaps'
outmapdir = '/Users/tunder/Dropbox/pagedata/pagemaps'
outtokendir = '/Users/tunder/Dropbox/pagedata/pagefeatures'
tokenfiles = os.listdir(tokendir)
tokendict = dict()
for filename in tokenfiles:
    if filename.endswith('pg.tsv'):
        inpath = tokendir + '/' + filename
        with open(inpath, encoding='utf-8') as f:
            filelines = f.readlines()
        tokendict[filename] = filelines
mapfiles = os.listdir(mapdir)
mapdict = dict()
for filename in mapfiles:
    if not filename.startswith('.'):
        inpath = mapdir + '/' + filename
        with open(inpath, encoding='utf-8') as f:
            filelines = f.readlines()
        mapdict[filename] = filelines
for filename, tokenlines in tokendict.items():
    strippedname = filename[:-11]
    if strippedname not in mapdict:
        print(strippedname + ' is missing.')
        continue
    maplines = mapdict[strippedname]
    striplasttwo = False
    if filename.startswith('mdp'):
        trailers = 0
        for line in maplines[-2::1]:
            line = line.rstrip()
            fields = line.split('\t')
            if fields[-1] == 'errat' or fields[-1] == 'libra':
                trailers += 1
        lastpage = 0
        for line in reversed(tokenlines):
            line = line.rstrip()
            fields = line.split('\t')
            if int(fields[0]) > lastpage:
                lastpage = int(fields[0])
            if int(fields[0]) + 2 < lastpage:
                break
            elif fields[1] == 'percentage' or fields[1] == 'implicit':
                trailers += 1
        if trailers > 3:
            striplasttwo = True
    lastpage = 0
    for line in maplines[1:]:
        fields = line.split('\t')
        if int(fields[0]) > lastpage:
            lastpage = int(fields[0])
    otherlastpage = 0
    for line in tokenlines:
        fields = line.split('\t')
        if int(fields[0]) > otherlastpage:
            otherlastpage = int(fields[0])
    if lastpage != otherlastpage:
        print(strippedname + ' ' + str(lastpage) + ' ' + str(otherlastpage))
    if striplasttwo:
        lastpage = lastpage - 2
        print(filename)
    outdatapath = outtokendir + '/' + strippedname + '.pg.tsv'
    with open(outdatapath, mode='w', encoding='utf-8') as f:
        for line in tokenlines:
            fields = line.split('\t')
            try:
                pagenumber = int(fields[0])
            except:
                pagenumber = 0
            if pagenumber <= lastpage:
                f.write(line)
    outmappath = outmapdir + '/' + strippedname + '.map'
    with open(outmappath, mode='w', encoding='utf-8') as f:
        for line in maplines:
            fields = line.split('\t')
            try:
                pagenumber = int(fields[0])
            except:
                pagenumber = 0
            if pagenumber <= lastpage:
                f.write(line)

# Collate training data

# So, we have genre maps generated by the pagetagger, as well as
# page-level feature/token counts generated by the new workflow in
# PythonScripts/workflow.

# The function of this script is to collate the two and copy them
# over to pagedata, making sure that we only include files where we
# have matches on both sides. Also, in the process, I need to trim
# the last two pages from certain mdp files that mistakenly included
# pages for "notes" and "pagedata" that were really metadata.

import os

tokendir = "/Users/tunder/Dropbox/PythonScripts/workflow/trainslice"

mapdir = "/Users/tunder/Dropbox/pagedata/detailedmaps"

outmapdir = "/Users/tunder/Dropbox/pagedata/pagemaps"
outtokendir = "/Users/tunder/Dropbox/pagedata/pagefeatures"

tokenfiles = os.listdir(tokendir)
tokendict = dict()

for filename in tokenfiles:
	if filename.endswith("pg.tsv"):
		inpath = tokendir + "/" + filename
		with open(inpath, encoding = 'utf-8') as f:
			filelines = f.readlines()
		tokendict[filename] = filelines

mapfiles = os.listdir(mapdir)
mapdict = dict()

for filename in mapfiles:
	if not filename.startswith("."):
		inpath = mapdir + "/" + filename
		with open(inpath, encoding = 'utf-8') as f:
			filelines = f.readlines()
		mapdict[filename] = filelines

for filename, tokenlines in tokendict.items():
	strippedname = filename[:-11]
	if strippedname not in mapdict:
		print(strippedname + " is missing.")
		continue

	maplines = mapdict[strippedname]

	striplasttwo = False

	if filename.startswith("mdp"):
		trailers = 0
		for line in maplines[-2::1]:
			line = line.rstrip()
			fields = line.split('\t')
			if fields[-1] == "errat" or fields[-1] == "libra":
				trailers += 1

		lastpage = 0
		for line in reversed(tokenlines):
			line = line.rstrip()
			fields = line.split('\t')
			if int(fields[0]) > lastpage:
				lastpage = int(fields[0])
			if int(fields[0]) + 2 < lastpage:
				break
			else:
				if fields[1] == "percentage" or fields[1] == "implicit":
					trailers += 1

		if trailers > 3:
			striplasttwo = True

	lastpage = 0
	for line in maplines[1:]:
		fields = line.split('\t')
		if int(fields[0]) > lastpage:
			lastpage = int(fields[0])

	otherlastpage = 0
	for line in tokenlines:
		fields = line.split('\t')
		if int(fields[0]) > otherlastpage:
			otherlastpage = int(fields[0])

	if lastpage != otherlastpage:
		print(strippedname + " " + str(lastpage) + " " + str(otherlastpage))

	if striplasttwo:
		lastpage = lastpage - 2
		print(filename)

	outdatapath = outtokendir + "/" + strippedname + ".pg.tsv"
	with open(outdatapath, mode = "w", encoding = "utf-8") as f:
		for line in tokenlines:
			fields = line.split('\t')
			try:
				pagenumber = int(fields[0])
			except:
				pagenumber = 0

			if pagenumber <= lastpage:
				f.write(line)

	outmappath = outmapdir + "/" + strippedname + ".map"
	with open(outmappath, mode = "w", encoding = "utf-8") as f:
		for line in maplines:
			fields = line.split('\t')
			try:
				pagenumber = int(fields[0])
			except:
				pagenumber = 0

			if pagenumber <= lastpage:
				f.write(line)















