import pickle

# prepare an answer for chunker from IOB text file

def get_chunker_answer(input_file, output_file):
    answer = []  # array of dicts

    with open(input_file, 'r', encoding='utf-8') as f:
        sentence = {}
        chunk = []
        line_number = 0
        noun = 0
    
        for line in f:
            if len(line) <= 1:  # empty line
                if len(chunk) > 0:
                    sentence[noun] = chunk  # finish sentence
                answer.append(sentence)
                sentence = {}
                chunk = []
                line_number = 0  # start new sentence
                continue

            line_number += 1
            parsed_line = line.split('\t')
            pos = (parsed_line[1] == 'NOUN' or parsed_line[1] == 'PROPN')
            punct = (parsed_line[1] == 'PUNCT')

            if parsed_line[3][:-1] == 'B':   # beginning of a chunk
                if len(chunk) > 0:
                    sentence[noun] = chunk
                if not punct:  # delete punctuation
                    chunk = [line_number]
                else:
                    chunk = []
                if pos:
                    noun = line_number
            elif parsed_line[3][:-1] == 'I':   # middle of a chunk
                if not punct:
                    chunk.append(line_number)
                if pos:
                    noun = line_number
            else:   # between chunks
                if len(chunk) > 0:
                    sentence[noun] = chunk
                    chunk = []

        if len(chunk) > 0:
            sentence[noun] = chunk  # finish sentence
        answer.append(sentence)
                
    with open(output_file, 'wb') as of:
        for e in answer:
            pickle.dump(e, of)
        
    return

5865b87ac75a3e87313243276a372810475b5eaf
<mask token>

<mask token>


def get_chunker_answer(input_file, output_file):
    answer = []
    with open(input_file, 'r', encoding='utf-8') as f:
        sentence = {}
        chunk = []
        line_number = 0
        noun = 0
        for line in f:
            if len(line) <= 1:
                if len(chunk) > 0:
                    sentence[noun] = chunk
                answer.append(sentence)
                sentence = {}
                chunk = []
                line_number = 0
                continue
            line_number += 1
            parsed_line = line.split('\t')
            pos = parsed_line[1] == 'NOUN' or parsed_line[1] == 'PROPN'
            punct = parsed_line[1] == 'PUNCT'
            if parsed_line[3][:-1] == 'B':
                if len(chunk) > 0:
                    sentence[noun] = chunk
                if not punct:
                    chunk = [line_number]
                else:
                    chunk = []
                if pos:
                    noun = line_number
            elif parsed_line[3][:-1] == 'I':
                if not punct:
                    chunk.append(line_number)
                if pos:
                    noun = line_number
            elif len(chunk) > 0:
                sentence[noun] = chunk
                chunk = []
        if len(chunk) > 0:
            sentence[noun] = chunk
        answer.append(sentence)
    with open(output_file, 'wb') as of:
        for e in answer:
            pickle.dump(e, of)
    return

import pickle


def get_chunker_answer(input_file, output_file):
    answer = []
    with open(input_file, 'r', encoding='utf-8') as f:
        sentence = {}
        chunk = []
        line_number = 0
        noun = 0
        for line in f:
            if len(line) <= 1:
                if len(chunk) > 0:
                    sentence[noun] = chunk
                answer.append(sentence)
                sentence = {}
                chunk = []
                line_number = 0
                continue
            line_number += 1
            parsed_line = line.split('\t')
            pos = parsed_line[1] == 'NOUN' or parsed_line[1] == 'PROPN'
            punct = parsed_line[1] == 'PUNCT'
            if parsed_line[3][:-1] == 'B':
                if len(chunk) > 0:
                    sentence[noun] = chunk
                if not punct:
                    chunk = [line_number]
                else:
                    chunk = []
                if pos:
                    noun = line_number
            elif parsed_line[3][:-1] == 'I':
                if not punct:
                    chunk.append(line_number)
                if pos:
                    noun = line_number
            elif len(chunk) > 0:
                sentence[noun] = chunk
                chunk = []
        if len(chunk) > 0:
            sentence[noun] = chunk
        answer.append(sentence)
    with open(output_file, 'wb') as of:
        for e in answer:
            pickle.dump(e, of)
    return

import pickle

# prepare an answer for chunker from IOB text file

def get_chunker_answer(input_file, output_file):
    answer = []  # array of dicts

    with open(input_file, 'r', encoding='utf-8') as f:
        sentence = {}
        chunk = []
        line_number = 0
        noun = 0
    
        for line in f:
            if len(line) <= 1:  # empty line
                if len(chunk) > 0:
                    sentence[noun] = chunk  # finish sentence
                answer.append(sentence)
                sentence = {}
                chunk = []
                line_number = 0  # start new sentence
                continue

            line_number += 1
            parsed_line = line.split('\t')
            pos = (parsed_line[1] == 'NOUN' or parsed_line[1] == 'PROPN')
            punct = (parsed_line[1] == 'PUNCT')

            if parsed_line[3][:-1] == 'B':   # beginning of a chunk
                if len(chunk) > 0:
                    sentence[noun] = chunk
                if not punct:  # delete punctuation
                    chunk = [line_number]
                else:
                    chunk = []
                if pos:
                    noun = line_number
            elif parsed_line[3][:-1] == 'I':   # middle of a chunk
                if not punct:
                    chunk.append(line_number)
                if pos:
                    noun = line_number
            else:   # between chunks
                if len(chunk) > 0:
                    sentence[noun] = chunk
                    chunk = []

        if len(chunk) > 0:
            sentence[noun] = chunk  # finish sentence
        answer.append(sentence)
                
    with open(output_file, 'wb') as of:
        for e in answer:
            pickle.dump(e, of)
        
    return
